diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 18:25:58 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 18:25:58 +0000 |
| commit | a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 (patch) | |
| tree | fb69158581673816a8cd895f9d352dcb3c678b1e /app/models | |
| parent | d16b2e8639e99961de6ddc93909f3bb5c1445ba1 (diff) | |
| download | gitlab-ce-a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42
Diffstat (limited to 'app/models')
206 files changed, 4633 insertions, 4060 deletions
diff --git a/app/models/analytics/cycle_analytics/project_level.rb b/app/models/analytics/cycle_analytics/project_level.rb new file mode 100644 index 00000000000..7a73bc75ed6 --- /dev/null +++ b/app/models/analytics/cycle_analytics/project_level.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class ProjectLevel + attr_reader :project, :options + + def initialize(project:, options:) + @project = project + @options = options.merge(project: project) + end + + def summary + @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new(project, + options: options, + current_user: options[:current_user]).data + end + + def permissions(user:) + Gitlab::CycleAnalytics::Permissions.get(user: user, project: project) + end + + def stats + @stats ||= default_stage_names.map do |stage_name| + self[stage_name].as_json + end + end + + def [](stage_name) + ::CycleAnalytics::ProjectLevelStageAdapter.new(build_stage(stage_name), options) + end + + private + + def build_stage(stage_name) + stage_params = stage_params_by_name(stage_name).merge(project: project) + Analytics::CycleAnalytics::ProjectStage.new(stage_params) + end + + def stage_params_by_name(name) + Gitlab::Analytics::CycleAnalytics::DefaultStages.find_by_name!(name) + end + + def default_stage_names + Gitlab::Analytics::CycleAnalytics::DefaultStages.symbolized_stage_names + end + end + end +end diff --git a/app/models/analytics/usage_trends/measurement.rb b/app/models/analytics/usage_trends/measurement.rb index 46c5d56d210..02e239ca0ef 100644 --- a/app/models/analytics/usage_trends/measurement.rb +++ b/app/models/analytics/usage_trends/measurement.rb @@ -3,7 +3,7 @@ module Analytics module UsageTrends class Measurement < ApplicationRecord - self.table_name = 'analytics_instance_statistics_measurements' + self.table_name = 'analytics_usage_trends_measurements' enum identifier: { projects: 1, diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 5e5bc00458e..a93348a3b27 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -53,10 +53,12 @@ class ApplicationRecord < ActiveRecord::Base # Start a new transaction with a shorter-than-usual statement timeout. This is # currently one third of the default 15-second timeout def self.with_fast_read_statement_timeout(timeout_ms = 5000) - transaction(requires_new: true) do - connection.exec_query("SET LOCAL statement_timeout = #{timeout_ms}") + ::Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do + transaction(requires_new: true) do + connection.exec_query("SET LOCAL statement_timeout = #{timeout_ms}") - yield + yield + end end end @@ -85,5 +87,3 @@ class ApplicationRecord < ActiveRecord::Base enum(enum_mod.key => values) end end - -ApplicationRecord.prepend_mod_with('ApplicationRecordHelpers') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 65800e40d6c..f8047ed9b78 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -273,6 +273,18 @@ class ApplicationSetting < ApplicationRecord greater_than_or_equal_to: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, less_than_or_equal_to: Gitlab::Git::Diff::MAX_PATCH_BYTES_UPPER_BOUND } + validates :diff_max_files, + presence: true, + numericality: { only_integer: true, + greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, + less_than_or_equal_to: Commit::MAX_DIFF_FILES_SETTING_UPPER_BOUND } + + validates :diff_max_lines, + presence: true, + numericality: { only_integer: true, + greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, + less_than_or_equal_to: Commit::MAX_DIFF_LINES_SETTING_UPPER_BOUND } + validates :user_default_internal_regex, js_regex: true, allow_nil: true validates :personal_access_token_prefix, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index bf9df3b9efc..b613e698471 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -60,6 +60,8 @@ module ApplicationSettingImplementation default_projects_limit: Settings.gitlab['default_projects_limit'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, + diff_max_files: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, + diff_max_lines: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, disable_feed_token: false, disabled_oauth_sign_in_sources: [], dns_rebinding_protection_enabled: true, diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 59ca4dbfec6..371b58dea03 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -4,6 +4,10 @@ module BulkImports class Export < ApplicationRecord include Gitlab::Utils::StrongMemoize + STARTED = 0 + FINISHED = 1 + FAILED = -1 + self.table_name = 'bulk_import_exports' belongs_to :project, optional: true @@ -18,9 +22,9 @@ module BulkImports validate :portable_relation? state_machine :status, initial: :started do - state :started, value: 0 - state :finished, value: 1 - state :failed, value: -1 + state :started, value: STARTED + state :finished, value: FINISHED + state :failed, value: FAILED event :start do transition any => :started diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb new file mode 100644 index 00000000000..98804d18f27 --- /dev/null +++ b/app/models/bulk_imports/export_status.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module BulkImports + class ExportStatus + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline_tracker, relation) + @pipeline_tracker = pipeline_tracker + @relation = relation + @entity = @pipeline_tracker.entity + @configuration = @entity.bulk_import.configuration + @client = Clients::HTTP.new(uri: @configuration.url, token: @configuration.access_token) + end + + def started? + export_status['status'] == Export::STARTED + end + + def failed? + export_status['status'] == Export::FAILED + end + + def error + export_status['error'] + end + + private + + attr_reader :client, :entity, :relation + + def export_status + strong_memoize(:export_status) do + fetch_export_status.find { |item| item['relation'] == relation } + end + rescue StandardError => e + { 'status' => Export::FAILED, 'error' => e.message } + end + + def fetch_export_status + client.get(status_endpoint).parsed_response + end + + def status_endpoint + "/groups/#{entity.encoded_source_full_path}/export_relations/status" + end + end +end diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index bb04e84ad72..7396f9d3655 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -13,6 +13,14 @@ module BulkImports attributes_finder.find_root(portable_class_sym) end + def top_relation_tree(relation) + portable_relations_tree[relation.to_s] + end + + def relation_excluded_keys(relation) + attributes_finder.find_excluded_keys(relation) + end + def export_path strong_memoize(:export_path) do relative_path = File.join(base_export_path, SecureRandom.hex) @@ -47,6 +55,10 @@ module BulkImports @portable_class_sym ||= portable_class.to_s.demodulize.underscore.to_sym end + def portable_relations_tree + @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys + end + def import_export_yaml raise NotImplementedError end diff --git a/app/models/chat_team.rb b/app/models/chat_team.rb index 6e39d7e2204..ee786ae6cb7 100644 --- a/app/models/chat_team.rb +++ b/app/models/chat_team.rb @@ -7,8 +7,8 @@ class ChatTeam < ApplicationRecord belongs_to :namespace def remove_mattermost_team(current_user) - Mattermost::Team.new(current_user).destroy(team_id: team_id) - rescue Mattermost::ClientError => e + ::Mattermost::Team.new(current_user).destroy(team_id: team_id) + rescue ::Mattermost::ClientError => e # Either the group is not found, or the user doesn't have the proper # access on the mattermost instance. In the first case, we're done either way # in the latter case, we can't recover by retrying, so we just log what happened diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 352229c64da..577bca282ef 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -163,7 +163,7 @@ module Ci def expanded_environment_name end - def instantized_environment + def persisted_environment end def execute_hooks diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 46fc87a6ea8..fdfffd9b0cd 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -11,6 +11,7 @@ module Ci include Importable include Ci::HasRef include IgnorableColumns + include TaggableQueries BuildArchivedError = Class.new(StandardError) @@ -37,6 +38,8 @@ module Ci has_one :deployment, as: :deployable, class_name: 'Deployment' has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build + has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id + has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build @@ -88,16 +91,6 @@ module Ci end end - # Initializing an object instead of fetching `persisted_environment` for avoiding unnecessary queries. - # We're planning to introduce a direct relationship between build and environment - # in https://gitlab.com/gitlab-org/gitlab/-/issues/326445 to let us to preload - # in batch. - def instantized_environment - return unless has_environment? - - ::Environment.new(project: self.project, name: self.expanded_environment_name) - end - serialize :options # rubocop:disable Cop/ActiveRecordSerialize serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiveRecordSerialize @@ -212,6 +205,8 @@ module Ci end scope :with_coverage, -> { where.not(coverage: nil) } + scope :without_coverage, -> { where(coverage: nil) } + scope :with_coverage_regex, -> { where.not(coverage_regex: nil) } scope :for_project, -> (project_id) { where(project_id: project_id) } @@ -222,6 +217,8 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } + after_save :stick_build_if_status_changed + after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end @@ -304,12 +301,35 @@ module Ci end end - after_transition any => [:pending] do |build| + # rubocop:disable CodeReuse/ServiceClass + after_transition any => [:pending] do |build, transition| + Ci::UpdateBuildQueueService.new.push(build, transition) + build.run_after_commit do BuildQueueWorker.perform_async(id) end end + after_transition pending: any do |build, transition| + Ci::UpdateBuildQueueService.new.pop(build, transition) + end + + after_transition any => [:running] do |build, transition| + Ci::UpdateBuildQueueService.new.track(build, transition) + end + + after_transition running: any do |build, transition| + Ci::UpdateBuildQueueService.new.untrack(build, transition) + + Ci::BuildRunnerSession.where(build: build).delete_all + end + + # rubocop:enable CodeReuse/ServiceClass + # + after_transition pending: :running do |build| + build.ensure_metadata.update_timeout_state + end + after_transition pending: :running do |build| build.deployment&.run @@ -362,14 +382,6 @@ module Ci end end - after_transition pending: :running do |build| - build.ensure_metadata.update_timeout_state - end - - after_transition running: any do |build| - Ci::BuildRunnerSession.where(build: build).delete_all - end - after_transition any => [:skipped, :canceled] do |build, transition| if transition.to_name == :skipped build.deployment&.skip @@ -379,6 +391,33 @@ module Ci end end + def self.build_matchers(project) + unique_params = [ + :protected, + Arel.sql("(#{arel_tag_names_array.to_sql})") + ] + + group(*unique_params).pluck('array_agg(id)', *unique_params).map do |values| + Gitlab::Ci::Matching::BuildMatcher.new({ + build_ids: values[0], + protected: values[1], + tag_list: values[2], + project: project + }) + end + end + + def build_matcher + strong_memoize(:build_matcher) do + Gitlab::Ci::Matching::BuildMatcher.new({ + protected: protected?, + tag_list: tag_list, + build_ids: [id], + project: project + }) + end + end + def auto_retry_allowed? auto_retry.allowed? end @@ -442,7 +481,13 @@ module Ci end def retryable? - !archived? && (success? || failed? || canceled?) + if Feature.enabled?(:prevent_retry_of_retried_jobs, project, default_enabled: :yaml) + return false if retried? || archived? + + success? || failed? || canceled? + else + !archived? && (success? || failed? || canceled?) + end end def retries_count @@ -560,6 +605,8 @@ module Ci variables.concat(persisted_environment.predefined_variables) + variables.append(key: 'CI_ENVIRONMENT_ACTION', value: environment_action) + # Here we're passing unexpanded environment_url for runner to expand, # and we need to make sure that CI_ENVIRONMENT_NAME and # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. @@ -716,22 +763,14 @@ module Ci end def any_runners_online? - if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) - cache_for_online_runners do - project.any_online_runners? { |runner| runner.match_build_if_online?(self) } - end - else - project.any_active_runners? { |runner| runner.match_build_if_online?(self) } + cache_for_online_runners do + project.any_online_runners? { |runner| runner.match_build_if_online?(self) } end end def any_runners_available? - if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) - cache_for_available_runners do - project.active_runners.exists? - end - else - project.any_active_runners? + cache_for_available_runners do + project.active_runners.exists? end end @@ -1039,6 +1078,28 @@ module Ci options.dig(:allow_failure_criteria, :exit_codes).present? end + def create_queuing_entry! + ::Ci::PendingBuild.upsert_from_build!(self) + end + + ## + # We can have only one queuing entry or running build tracking entry, + # because there is a unique index on `build_id` in each table, but we need + # a relation to remove these entries more efficiently in a single statement + # without actually loading data. + # + def all_queuing_entries + ::Ci::PendingBuild.where(build_id: self.id) + end + + def all_runtime_metadata + ::Ci::RunningBuild.where(build_id: self.id) + end + + def shared_runner_build? + runner&.instance_type? + end + protected def run_status_commit_hooks! @@ -1049,6 +1110,13 @@ module Ci private + def stick_build_if_status_changed + return unless saved_change_to_status? + return unless running? + + ::Gitlab::Database::LoadBalancing::Sticking.stick(:build, id) + end + def status_commit_hooks @status_commit_hooks ||= [] end diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb index 716d919487d..d39e0411a79 100644 --- a/app/models/ci/build_dependencies.rb +++ b/app/models/ci/build_dependencies.rb @@ -143,8 +143,6 @@ module Ci def specified_cross_pipeline_dependencies strong_memoize(:specified_cross_pipeline_dependencies) do - next [] unless Feature.enabled?(:ci_cross_pipeline_artifacts_download, processable.project, default_enabled: true) - specified_cross_dependencies.select { |dep| dep[:pipeline] && dep[:artifacts] } end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 4094bdb26dc..bb2dac5cd43 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -10,6 +10,7 @@ module Ci include Presentable include ChronicDurationAttribute include Gitlab::Utils::StrongMemoize + include IgnorableColumns self.table_name = 'ci_builds_metadata' @@ -21,8 +22,8 @@ module Ci validates :build, presence: true validates :secrets, json_schema: { filename: 'build_metadata_secrets' } - serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize - serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_options, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_variables, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize chronic_duration_attr_reader :timeout_human_readable, :timeout @@ -37,6 +38,8 @@ module Ci job_timeout_source: 4 } + ignore_column :build_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' + def update_timeout_state timeout = timeout_with_highest_precedence diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 719511bbb8a..25f4a06088d 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -14,7 +14,13 @@ module Ci belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id - default_value_for :data_store, :redis + default_value_for :data_store do + if Feature.enabled?(:dedicated_redis_trace_chunks, type: :ops) + :redis_trace_chunks + else + :redis + end + end after_create { metrics.increment_trace_operation(operation: :chunked) } @@ -25,22 +31,22 @@ module Ci FailedToPersistDataError = Class.new(StandardError) - # Note: The ordering of this hash is related to the precedence of persist store. - # The bottom item takes the highest precedence, and the top item takes the lowest precedence. DATA_STORES = { redis: 1, database: 2, - fog: 3 + fog: 3, + redis_trace_chunks: 4 }.freeze STORE_TYPES = DATA_STORES.keys.to_h do |store| - [store, "Ci::BuildTraceChunks::#{store.capitalize}".constantize] + [store, "Ci::BuildTraceChunks::#{store.to_s.camelize}".constantize] end.freeze + LIVE_STORES = %i[redis redis_trace_chunks].freeze enum data_store: DATA_STORES - scope :live, -> { redis } - scope :persisted, -> { not_redis.order(:chunk_index) } + scope :live, -> { where(data_store: LIVE_STORES) } + scope :persisted, -> { where.not(data_store: LIVE_STORES).order(:chunk_index) } class << self def all_stores @@ -48,8 +54,7 @@ module Ci end def persistable_store - # get first available store from the back of the list - all_stores.reverse.find { |store| get_store_class(store).available? } + STORE_TYPES[:fog].available? ? :fog : :database end def get_store_class(store) @@ -85,16 +90,10 @@ module Ci # change the behavior in CE. # def with_read_consistency(build, &block) - return yield unless consistent_reads_enabled?(build) - ::Gitlab::Database::Consistency .with_read_consistency(&block) end - def consistent_reads_enabled?(build) - Feature.enabled?(:gitlab_ci_trace_read_consistency, build.project, type: :development, default_enabled: true) - end - ## # Sometimes we do not want to read raw data. This method makes it easier # to find attributes that are just metadata excluding raw data. @@ -201,7 +200,7 @@ module Ci end def flushed? - !redis? + !live? end def migrated? @@ -209,7 +208,7 @@ module Ci end def live? - redis? + LIVE_STORES.include?(data_store.to_sym) end def <=>(other) diff --git a/app/models/ci/build_trace_chunks/database.rb b/app/models/ci/build_trace_chunks/database.rb index 7448afba4c2..895028778a9 100644 --- a/app/models/ci/build_trace_chunks/database.rb +++ b/app/models/ci/build_trace_chunks/database.rb @@ -3,10 +3,6 @@ module Ci module BuildTraceChunks class Database - def available? - true - end - def keys(relation) [] end diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index cbf0c0a1696..fab85fae33d 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -3,10 +3,18 @@ module Ci module BuildTraceChunks class Fog - def available? + def self.available? object_store.enabled end + def self.object_store + Gitlab.config.artifacts.object_store + end + + def available? + self.class.available? + end + def data(model) files.get(key(model))&.body rescue Excon::Error::NotFound @@ -85,7 +93,7 @@ module Ci end def object_store - Gitlab.config.artifacts.object_store + self.class.object_store end def object_store_raw_config diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 003ec107895..46f275636e1 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -2,92 +2,11 @@ module Ci module BuildTraceChunks - class Redis - CHUNK_REDIS_TTL = 1.week - LUA_APPEND_CHUNK = <<~EOS - local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] - local length = new_data:len() - local expire = #{CHUNK_REDIS_TTL.seconds} - local current_size = redis.call("strlen", key) - offset = tonumber(offset) - - if offset == 0 then - -- overwrite everything - redis.call("set", key, new_data, "ex", expire) - return redis.call("strlen", key) - elseif offset > current_size then - -- offset range violation - return -1 - elseif offset + length >= current_size then - -- efficiently append or overwrite and append - redis.call("expire", key, expire) - return redis.call("setrange", key, offset, new_data) - else - -- append and truncate - local current_data = redis.call("get", key) - new_data = current_data:sub(1, offset) .. new_data - redis.call("set", key, new_data, "ex", expire) - return redis.call("strlen", key) - end - EOS - - def available? - true - end - - def data(model) - Gitlab::Redis::SharedState.with do |redis| - redis.get(key(model)) - end - end - - def set_data(model, new_data) - Gitlab::Redis::SharedState.with do |redis| - redis.set(key(model), new_data, ex: CHUNK_REDIS_TTL) - end - end - - def append_data(model, new_data, offset) - Gitlab::Redis::SharedState.with do |redis| - redis.eval(LUA_APPEND_CHUNK, keys: [key(model)], argv: [new_data, offset]) - end - end - - def size(model) - Gitlab::Redis::SharedState.with do |redis| - redis.strlen(key(model)) - end - end - - def delete_data(model) - delete_keys([[model.build_id, model.chunk_index]]) - end - - def keys(relation) - relation.pluck(:build_id, :chunk_index) - end - - def delete_keys(keys) - return if keys.empty? - - keys = keys.map { |key| key_raw(*key) } - - Gitlab::Redis::SharedState.with do |redis| - # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.del(keys) - end - end - end - + class Redis < RedisBase private - def key(model) - key_raw(model.build_id, model.chunk_index) - end - - def key_raw(build_id, chunk_index) - "gitlab:ci:trace:#{build_id.to_i}:chunks:#{chunk_index.to_i}" + def with_redis + Gitlab::Redis::SharedState.with { |redis| yield(redis) } end end end diff --git a/app/models/ci/build_trace_chunks/redis_base.rb b/app/models/ci/build_trace_chunks/redis_base.rb new file mode 100644 index 00000000000..3b7a844d122 --- /dev/null +++ b/app/models/ci/build_trace_chunks/redis_base.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Ci + module BuildTraceChunks + class RedisBase + CHUNK_REDIS_TTL = 1.week + LUA_APPEND_CHUNK = <<~EOS + local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] + local length = new_data:len() + local expire = #{CHUNK_REDIS_TTL.seconds} + local current_size = redis.call("strlen", key) + offset = tonumber(offset) + + if offset == 0 then + -- overwrite everything + redis.call("set", key, new_data, "ex", expire) + return redis.call("strlen", key) + elseif offset > current_size then + -- offset range violation + return -1 + elseif offset + length >= current_size then + -- efficiently append or overwrite and append + redis.call("expire", key, expire) + return redis.call("setrange", key, offset, new_data) + else + -- append and truncate + local current_data = redis.call("get", key) + new_data = current_data:sub(1, offset) .. new_data + redis.call("set", key, new_data, "ex", expire) + return redis.call("strlen", key) + end + EOS + + def data(model) + with_redis do |redis| + redis.get(key(model)) + end + end + + def set_data(model, new_data) + with_redis do |redis| + redis.set(key(model), new_data, ex: CHUNK_REDIS_TTL) + end + end + + def append_data(model, new_data, offset) + with_redis do |redis| + redis.eval(LUA_APPEND_CHUNK, keys: [key(model)], argv: [new_data, offset]) + end + end + + def size(model) + with_redis do |redis| + redis.strlen(key(model)) + end + end + + def delete_data(model) + delete_keys([[model.build_id, model.chunk_index]]) + end + + def keys(relation) + relation.pluck(:build_id, :chunk_index) + end + + def delete_keys(keys) + return if keys.empty? + + keys = keys.map { |key| key_raw(*key) } + + with_redis do |redis| + # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.del(keys) + end + end + end + + private + + def key(model) + key_raw(model.build_id, model.chunk_index) + end + + def key_raw(build_id, chunk_index) + "gitlab:ci:trace:#{build_id.to_i}:chunks:#{chunk_index.to_i}" + end + end + end +end diff --git a/app/models/ci/build_trace_chunks/redis_trace_chunks.rb b/app/models/ci/build_trace_chunks/redis_trace_chunks.rb new file mode 100644 index 00000000000..06e315b0aaf --- /dev/null +++ b/app/models/ci/build_trace_chunks/redis_trace_chunks.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + module BuildTraceChunks + class RedisTraceChunks < RedisBase + private + + def with_redis + Gitlab::Redis::TraceChunks.with { |redis| yield(redis) } + end + end + end +end diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb index 5091e3ff04a..036f611a61c 100644 --- a/app/models/ci/build_trace_section.rb +++ b/app/models/ci/build_trace_section.rb @@ -4,11 +4,14 @@ module Ci class BuildTraceSection < ApplicationRecord extend SuppressCompositePrimaryKeyWarning extend Gitlab::Ci::Model + include IgnorableColumns belongs_to :build, class_name: 'Ci::Build' belongs_to :project belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName' validates :section_name, :build, :project, presence: true, allow_blank: false + + ignore_column :build_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 5248a80f710..6a7a2b3f6bd 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -18,7 +18,6 @@ module Ci ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze NON_ERASABLE_FILE_TYPES = %w[trace].freeze TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze - UNSUPPORTED_FILE_TYPES = %i[license_management].freeze SAST_REPORT_TYPES = %w[sast].freeze SECRET_DETECTION_REPORT_TYPES = %w[secret_detection].freeze DEFAULT_FILE_NAMES = { @@ -35,7 +34,6 @@ module Ci dependency_scanning: 'gl-dependency-scanning-report.json', container_scanning: 'gl-container-scanning-report.json', dast: 'gl-dast-report.json', - license_management: 'gl-license-management-report.json', license_scanning: 'gl-license-scanning-report.json', performance: 'performance.json', browser_performance: 'browser-performance.json', @@ -45,7 +43,7 @@ module Ci dotenv: '.env', cobertura: 'cobertura-coverage.xml', terraform: 'tfplan.json', - cluster_applications: 'gl-cluster-applications.json', + cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 requirements: 'requirements.json', coverage_fuzzing: 'gl-coverage-fuzzing.json', api_fuzzing: 'gl-api-fuzzing-report.json' @@ -74,7 +72,6 @@ module Ci dependency_scanning: :raw, container_scanning: :raw, dast: :raw, - license_management: :raw, license_scanning: :raw, # All these file formats use `raw` as we need to store them uncompressed @@ -102,7 +99,6 @@ module Ci dependency_scanning dotenv junit - license_management license_scanning lsif metrics @@ -124,7 +120,6 @@ module Ci mount_file_store_uploader JobArtifactUploader validates :file_format, presence: true, unless: :trace?, on: :create - validate :validate_supported_file_format!, on: :create validate :validate_file_format!, unless: :trace?, on: :create before_save :set_size, if: :file_changed? @@ -199,8 +194,7 @@ module Ci container_scanning: 7, ## EE-specific dast: 8, ## EE-specific codequality: 9, ## EE-specific - license_management: 10, ## EE-specific - license_scanning: 101, ## EE-specific till 13.0 + license_scanning: 101, ## EE-specific performance: 11, ## EE-specific till 13.2 metrics: 12, ## EE-specific metrics_referee: 13, ## runner referees @@ -233,14 +227,6 @@ module Ci hashed_path: 2 } - def validate_supported_file_format! - return if Feature.disabled?(:drop_license_management_artifact, project, default_enabled: true) - - if UNSUPPORTED_FILE_TYPES.include?(self.file_type&.to_sym) - errors.add(:base, _("File format is no longer supported")) - end - end - def validate_file_format! unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym errors.add(:base, _('Invalid file format with specified file type')) diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb new file mode 100644 index 00000000000..283ad4a190d --- /dev/null +++ b/app/models/ci/job_token/project_scope_link.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# The connection between a source project (which defines the job token scope) +# and a target project which is the one allowed to be accessed by the job token. + +module Ci + module JobToken + class ProjectScopeLink < ApplicationRecord + self.table_name = 'ci_job_token_project_scope_links' + + belongs_to :source_project, class_name: 'Project' + belongs_to :target_project, class_name: 'Project' + belongs_to :added_by, class_name: 'User' + + scope :from_project, ->(project) { where(source_project: project) } + scope :to_project, ->(project) { where(target_project: project) } + + validates :source_project, presence: true + validates :target_project, presence: true + validate :not_self_referential_link + + private + + def not_self_referential_link + return unless source_project && target_project + + if source_project == target_project + self.errors.add(:target_project, _("can't be the same as the source project")) + end + end + end + end +end diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb new file mode 100644 index 00000000000..42cfdc21d66 --- /dev/null +++ b/app/models/ci/job_token/scope.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# This model represents the surface where a CI_JOB_TOKEN can be used. +# A Scope is initialized with the project that the job token belongs to, +# and indicates what are all the other projects that the token could access. +# +# By default a job token can only access its own project, which is the same +# project that defines the scope. +# By adding ScopeLinks to the scope we can allow other projects to be accessed +# by the job token. This works as an allowlist of projects for a job token. +# +# If a project is not included in the scope we should not allow the job user +# to access it since operations using CI_JOB_TOKEN should be considered untrusted. + +module Ci + module JobToken + class Scope + attr_reader :source_project + + def initialize(project) + @source_project = project + end + + def includes?(target_project) + # if the setting is disabled any project is considered to be in scope. + return true unless source_project.ci_job_token_scope_enabled? + + target_project.id == source_project.id || + Ci::JobToken::ProjectScopeLink.from_project(source_project).to_project(target_project).exists? + end + + def all_projects + Project.from_union([ + Project.id_in(source_project), + Project.where_exists( + Ci::JobToken::ProjectScopeLink + .from_project(source_project) + .where('projects.id = ci_job_token_project_scope_links.target_project_id')) + ], remove_duplicates: false) + end + end + end +end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb new file mode 100644 index 00000000000..b9a8a44bd6b --- /dev/null +++ b/app/models/ci/pending_build.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class PendingBuild < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :build, class_name: 'Ci::Build' + + def self.upsert_from_build!(build) + entry = self.new(build: build, project: build.project, protected: build.protected?) + + entry.validate! + + self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index f0a2c074584..ae06bea5a02 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -644,6 +644,10 @@ module Ci end end + def update_builds_coverage + builds.with_coverage_regex.without_coverage.each(&:update_coverage) + end + def batch_lookup_report_artifact_for_file_type(file_type) latest_report_artifacts .values_at(*::Ci::JobArtifact.associated_file_types_for(file_type.to_s)) @@ -660,15 +664,9 @@ module Ci # Return a hash of file type => array of 1 job artifact def latest_report_artifacts ::Gitlab::SafeRequestStore.fetch("pipeline:#{self.id}:latest_report_artifacts") do - # Note we use read_attribute(:project_id) to read the project - # ID instead of self.project_id. The latter appears to load - # the Project model. This extra filter doesn't appear to - # affect query plan but included to ensure we don't leak the - # wrong informaiton. ::Ci::JobArtifact.where( id: job_artifacts.with_reports .select('max(ci_job_artifacts.id) as id') - .where(project_id: self.read_attribute(:project_id)) .group(:file_type) ) .preload(:job) @@ -928,6 +926,12 @@ module Ci Ci::Build.latest.where(pipeline: self_and_descendants) end + def environments_in_self_and_descendants + environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id') + + Environment.where(id: environment_ids) + end + # Without using `unscoped`, caller scope is also included into the query. # Using `unscoped` here will be redundant after Rails 6.1 def self_and_descendants @@ -1252,6 +1256,10 @@ module Ci end end + def build_matchers + self.builds.build_matchers(project) + end + private def add_message(severity, content) diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 9e5d517c1fe..effe2d95a99 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -3,6 +3,7 @@ module Ci class PipelineSchedule < ApplicationRecord extend Gitlab::Ci::Model + extend ::Gitlab::Utils::Override include Importable include StripAttribute include CronSchedulable @@ -55,6 +56,17 @@ module Ci variables&.map(&:to_runner_variable) || [] end + override :set_next_run_at + def set_next_run_at + self.next_run_at = ::Ci::PipelineSchedules::CalculateNextRunService # rubocop: disable CodeReuse/ServiceClass + .new(project) + .execute(self, fallback_method: method(:calculate_next_run_at)) + end + + def daily_limit + project.actual_limits.limit_for(:ci_daily_pipeline_schedule_triggers) + end + private def worker_cron_expression diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 15c57550159..e2f257eab25 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -120,7 +120,7 @@ module Ci raise NotImplementedError end - def instantized_environment + def persisted_environment raise NotImplementedError end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8c877c2b818..71110ef0696 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -10,6 +10,8 @@ module Ci include TokenAuthenticatable include IgnorableColumns include FeatureGate + include Gitlab::Utils::StrongMemoize + include TaggableQueries add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption, default_enabled: true) ? :optional : :required } @@ -58,6 +60,7 @@ module Ci scope :active, -> { where(active: true) } scope :paused, -> { where(active: false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } + scope :recent, -> { where('ci_runners.created_at > :date OR ci_runners.contacted_at > :date', date: 3.months.ago) } # The following query using negation is cheaper than using `contacted_at <= ?` # because there are less runners online than have been created. The # resulting query is quickly finding online ones and then uses the regular @@ -131,6 +134,8 @@ module Ci end scope :order_contacted_at_asc, -> { order(contacted_at: :asc) } + scope :order_contacted_at_desc, -> { order(contacted_at: :desc) } + scope :order_created_at_asc, -> { order(created_at: :asc) } scope :order_created_at_desc, -> { order(created_at: :desc) } scope :with_tags, -> { preload(:tags) } @@ -161,20 +166,17 @@ module Ci numericality: { greater_than_or_equal_to: 0.0, message: 'needs to be non-negative' } + validates :config, json_schema: { filename: 'ci_runner_config' } + # Searches for runners matching the given query. # - # This method uses ILIKE on PostgreSQL. - # - # This method performs a *partial* match on tokens, thus a query for "a" - # will match any runner where the token contains the letter "a". As a result - # you should *not* use this method for non-admin purposes as otherwise users - # might be able to query a list of all runners. + # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. # # query - The search query as a String. # # Returns an ActiveRecord::Relation. def self.search(query) - fuzzy_search(query, [:token, :description]) + where(token: query).or(fuzzy_search(query, [:description])) end def self.online_contact_time_deadline @@ -190,13 +192,54 @@ module Ci end def self.order_by(order) - if order == 'contacted_asc' + case order + when 'contacted_asc' order_contacted_at_asc + when 'contacted_desc' + order_contacted_at_desc + when 'created_at_asc' + order_created_at_asc else order_created_at_desc end end + def self.runner_matchers + unique_params = [ + :runner_type, + :public_projects_minutes_cost_factor, + :private_projects_minutes_cost_factor, + :run_untagged, + :access_level, + Arel.sql("(#{arel_tag_names_array.to_sql})") + ] + + # we use distinct to de-duplicate data + distinct.pluck(*unique_params).map do |values| + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_type: values[0], + public_projects_minutes_cost_factor: values[1], + private_projects_minutes_cost_factor: values[2], + run_untagged: values[3], + access_level: values[4], + tag_list: values[5] + }) + end + end + + def runner_matcher + strong_memoize(:runner_matcher) do + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_type: runner_type, + public_projects_minutes_cost_factor: public_projects_minutes_cost_factor, + private_projects_minutes_cost_factor: private_projects_minutes_cost_factor, + run_untagged: run_untagged, + access_level: access_level, + tag_list: tag_list + }) + end + end + def assign_to(project, current_user = nil) if instance_type? self.runner_type = :project_type @@ -298,6 +341,14 @@ module Ci end def tick_runner_queue + ## + # We only stick a runner to primary database to be able to detect the + # replication lag in `EE::Ci::RegisterJobService#execute`. The + # intention here is not to execute `Ci::RegisterJobService#execute` on + # the primary database. + # + ::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id) + SecureRandom.hex.tap do |new_update| ::Gitlab::Workhorse.set_key_and_notify(runner_queue_key, new_update, expire: RUNNER_QUEUE_EXPIRY_TIME, overwrite: true) @@ -315,21 +366,24 @@ module Ci end def heartbeat(values) - values = values&.slice(:version, :revision, :platform, :architecture, :ip_address) || {} - values[:contacted_at] = Time.current + ## + # We can safely ignore writes performed by a runner heartbeat. We do + # not want to upgrade database connection proxy to use the primary + # database after heartbeat write happens. + # + ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do + values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config) || {} + values[:contacted_at] = Time.current - cache_attributes(values) + cache_attributes(values) - # We save data without validation, it will always change due to `contacted_at` - self.update_columns(values) if persist_cached_data? + # We save data without validation, it will always change due to `contacted_at` + self.update_columns(values) if persist_cached_data? + end end def pick_build!(build) - if Feature.enabled?(:ci_reduce_queries_when_ticking_runner_queue, self, default_enabled: :yaml) - tick_runner_queue if matches_build?(build) - else - tick_runner_queue if can_pick?(build) - end + tick_runner_queue if matches_build?(build) end def uncached_contacted_at @@ -395,13 +449,7 @@ module Ci end def matches_build?(build) - return false if self.ref_protected? && !build.protected? - - accepting_tags?(build) - end - - def accepting_tags?(build) - (run_untagged? || build.has_tags?) && (build.tag_list - tag_list).empty? + runner_matcher.matches?(build.build_matcher) end end end diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index f819dda207d..41a4c9012ff 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -7,6 +7,7 @@ module Ci self.limit_name = 'ci_registered_group_runners' self.limit_scope = :group + self.limit_relation = :recent_runners self.limit_feature_flag = :ci_runner_limits belongs_to :runner, inverse_of: :runner_namespaces @@ -16,6 +17,10 @@ module Ci validates :runner_id, uniqueness: { scope: :namespace_id } validate :group_runner_type + def recent_runners + ::Ci::Runner.belonging_to_group(namespace_id).recent + end + private def group_runner_type diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index c26b8183b52..af2595ce4af 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -7,11 +7,16 @@ module Ci self.limit_name = 'ci_registered_project_runners' self.limit_scope = :project + self.limit_relation = :recent_runners self.limit_feature_flag = :ci_runner_limits belongs_to :runner, inverse_of: :runner_projects belongs_to :project, inverse_of: :runner_projects + def recent_runners + ::Ci::Runner.belonging_to_project(project_id).recent + end + validates :runner_id, uniqueness: { scope: :project_id } end end diff --git a/app/models/ci/running_build.rb b/app/models/ci/running_build.rb new file mode 100644 index 00000000000..9446cfa05da --- /dev/null +++ b/app/models/ci/running_build.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Ci + class RunningBuild < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :build, class_name: 'Ci::Build' + belongs_to :runner, class_name: 'Ci::Runner' + + enum runner_type: ::Ci::Runner.runner_types + + def self.upsert_shared_runner_build!(build) + unless build.shared_runner_build? + raise ArgumentError, 'build has not been picked by a shared runner' + end + + entry = self.new(build: build, + project: build.project, + runner: build.runner, + runner_type: build.runner.runner_type) + + entry.validate! + + self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) + end + end +end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index ef920b2d589..d00066b778d 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -7,6 +7,9 @@ module Ci include Ci::HasStatus include Gitlab::OptimisticLocking include Presentable + include IgnorableColumns + + ignore_column :id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' enum status: Ci::HasStatus::STATUSES_ENUM diff --git a/app/models/clusters/applications/fluentd.rb b/app/models/clusters/applications/fluentd.rb deleted file mode 100644 index 91aa422b859..00000000000 --- a/app/models/clusters/applications/fluentd.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - class Fluentd < ApplicationRecord - VERSION = '2.4.0' - CILIUM_CONTAINER_NAME = 'cilium-monitor' - - self.table_name = 'clusters_applications_fluentd' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - default_value_for :version, VERSION - default_value_for :port, 514 - default_value_for :protocol, :tcp - - enum protocol: { tcp: 0, udp: 1 } - - validate :has_at_least_one_log_enabled? - - def chart - 'fluentd/fluentd' - end - - def repository - 'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive' - end - - def install_command - helm_command_module::InstallCommand.new( - name: 'fluentd', - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files - ) - end - - def values - content_values.to_yaml - end - - private - - def has_at_least_one_log_enabled? - if !waf_log_enabled && !cilium_log_enabled - errors.add(:base, _("At least one logging option is required to be enabled")) - end - end - - def content_values - YAML.load_file(chart_values_file).deep_merge!(specification) - end - - def specification - { - "configMaps" => { - "output.conf" => output_configuration_content, - "general.conf" => general_configuration_content - } - } - end - - def output_configuration_content - <<~EOF - <match kubernetes.**> - @type remote_syslog - @id out_kube_remote_syslog - host #{host} - port #{port} - program fluentd - hostname ${kubernetes_host} - protocol #{protocol} - packet_size 131072 - <buffer kubernetes_host> - </buffer> - <format> - @type ltsv - </format> - </match> - EOF - end - - def general_configuration_content - <<~EOF - <match fluent.**> - @type null - </match> - <source> - @type http - port 9880 - bind 0.0.0.0 - </source> - <source> - @type tail - @id in_tail_container_logs - path #{path_to_logs} - pos_file /var/log/fluentd-containers.log.pos - tag kubernetes.* - read_from_head true - <parse> - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ - </parse> - </source> - EOF - end - - def path_to_logs - path = [] - path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled - path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled - path.join(',') - end - end - end -end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index e7d4d737b8e..3a8c314efe4 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -7,10 +7,6 @@ module Clusters class Ingress < ApplicationRecord VERSION = '1.40.2' INGRESS_CONTAINER_NAME = 'nginx-ingress-controller' - MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log' - MODSECURITY_MODE_LOGGING = "DetectionOnly" - MODSECURITY_MODE_BLOCKING = "On" - MODSECURITY_OWASP_RULES_FILE = "/etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf" self.table_name = 'clusters_applications_ingress' @@ -20,22 +16,18 @@ module Clusters include ::Clusters::Concerns::ApplicationData include AfterCommitQueue include UsageStatistics + include IgnorableColumns default_value_for :ingress_type, :nginx - default_value_for :modsecurity_enabled, true default_value_for :version, VERSION - default_value_for :modsecurity_mode, :logging + + ignore_column :modsecurity_enabled, remove_with: '14.2', remove_after: '2021-07-22' + ignore_column :modsecurity_mode, remove_with: '14.2', remove_after: '2021-07-22' enum ingress_type: { nginx: 1 } - enum modsecurity_mode: { logging: 0, blocking: 1 } - - scope :modsecurity_not_installed, -> { where(modsecurity_enabled: nil) } - scope :modsecurity_enabled, -> { where(modsecurity_enabled: true) } - scope :modsecurity_disabled, -> { where(modsecurity_enabled: false) } - FETCH_IP_ADDRESS_DELAY = 30.seconds state_machine :status do @@ -92,96 +84,13 @@ module Clusters private - def specification - return {} unless modsecurity_enabled - - { - "controller" => { - "config" => { - "enable-modsecurity" => "true", - "enable-owasp-modsecurity-crs" => "false", - "modsecurity-snippet" => modsecurity_snippet_content, - "modsecurity.conf" => modsecurity_config_content - }, - "extraContainers" => [ - { - "name" => MODSECURITY_LOG_CONTAINER_NAME, - "image" => "busybox", - "args" => [ - "/bin/sh", - "-c", - "tail -F /var/log/modsec/audit.log" - ], - "volumeMounts" => [ - { - "name" => "modsecurity-log-volume", - "mountPath" => "/var/log/modsec", - "readOnly" => true - } - ], - "livenessProbe" => { - "exec" => { - "command" => [ - "ls", - "/var/log/modsec/audit.log" - ] - } - } - } - ], - "extraVolumeMounts" => [ - { - "name" => "modsecurity-template-volume", - "mountPath" => "/etc/nginx/modsecurity/modsecurity.conf", - "subPath" => "modsecurity.conf" - }, - { - "name" => "modsecurity-log-volume", - "mountPath" => "/var/log/modsec" - } - ], - "extraVolumes" => [ - { - "name" => "modsecurity-template-volume", - "configMap" => { - "name" => "ingress-#{INGRESS_CONTAINER_NAME}", - "items" => [ - { - "key" => "modsecurity.conf", - "path" => "modsecurity.conf" - } - ] - } - }, - { - "name" => "modsecurity-log-volume", - "emptyDir" => {} - } - ] - } - } - end - - def modsecurity_config_content - File.read(modsecurity_config_file_path) - end - - def modsecurity_config_file_path - Rails.root.join('vendor', 'ingress', 'modsecurity.conf') - end - def content_values - YAML.load_file(chart_values_file).deep_merge!(specification) + YAML.load_file(chart_values_file) end def application_jupyter_installed? cluster.application_jupyter&.installed? end - - def modsecurity_snippet_content - sec_rule_engine = logging? ? MODSECURITY_MODE_LOGGING : MODSECURITY_MODE_BLOCKING - "SecRuleEngine #{sec_rule_engine}\nInclude #{MODSECURITY_OWASP_RULES_FILE}" - end end end end diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 6867d7b6934..0e7cbb35e47 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -141,13 +141,13 @@ module Clusters end def install_knative_metrics - return [] unless cluster.application_prometheus_available? + return [] unless cluster.application_prometheus&.available? [Gitlab::Kubernetes::KubectlCmd.apply_file(METRICS_CONFIG)] end def delete_knative_istio_metrics - return [] unless cluster.application_prometheus_available? + return [] unless cluster.application_prometheus&.available? [Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "-f", METRICS_CONFIG)] end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index e8d56072b89..49840e3a2e7 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.28.0' + VERSION = '0.29.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 4877ced795c..2fff0a69a26 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -21,7 +21,6 @@ module Clusters Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter, Clusters::Applications::Knative.application_name => Clusters::Applications::Knative, Clusters::Applications::ElasticStack.application_name => Clusters::Applications::ElasticStack, - Clusters::Applications::Fluentd.application_name => Clusters::Applications::Fluentd, Clusters::Applications::Cilium.application_name => Clusters::Applications::Cilium }.freeze DEFAULT_ENVIRONMENT = '*' @@ -68,7 +67,6 @@ module Clusters has_one_cluster_application :jupyter has_one_cluster_application :knative has_one_cluster_application :elastic_stack - has_one_cluster_application :fluentd has_one_cluster_application :cilium has_many :kubernetes_namespaces @@ -104,8 +102,8 @@ module Clusters delegate :available?, to: :application_helm, prefix: true, allow_nil: true delegate :available?, to: :application_ingress, prefix: true, allow_nil: true delegate :available?, to: :application_knative, prefix: true, allow_nil: true - delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true + delegate :available?, to: :integration_prometheus, prefix: true, allow_nil: true delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true @@ -138,11 +136,10 @@ module Clusters scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) } scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) } - scope :with_enabled_modsecurity, -> { joins(:application_ingress).merge(::Clusters::Applications::Ingress.modsecurity_enabled) } scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) } scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) } scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct } - scope :preload_elasticstack, -> { preload(:application_elastic_stack) } + scope :preload_elasticstack, -> { preload(:integration_elastic_stack) } scope :preload_environments, -> { preload(:environments) } scope :managed, -> { where(managed: true) } @@ -171,18 +168,16 @@ module Clusters state_machine :cleanup_status, initial: :cleanup_not_started do state :cleanup_not_started, value: 1 - state :cleanup_uninstalling_applications, value: 2 state :cleanup_removing_project_namespaces, value: 3 state :cleanup_removing_service_account, value: 4 state :cleanup_errored, value: 5 event :start_cleanup do |cluster| - transition [:cleanup_not_started, :cleanup_errored] => :cleanup_uninstalling_applications + transition [:cleanup_not_started, :cleanup_errored] => :cleanup_removing_project_namespaces end event :continue_cleanup do transition( - cleanup_uninstalling_applications: :cleanup_removing_project_namespaces, cleanup_removing_project_namespaces: :cleanup_removing_service_account) end @@ -195,13 +190,7 @@ module Clusters cluster.cleanup_status_reason = status_reason if status_reason end - after_transition [:cleanup_not_started, :cleanup_errored] => :cleanup_uninstalling_applications do |cluster| - cluster.run_after_commit do - Clusters::Cleanup::AppWorker.perform_async(cluster.id) - end - end - - after_transition cleanup_uninstalling_applications: :cleanup_removing_project_namespaces do |cluster| + after_transition [:cleanup_not_started, :cleanup_errored] => :cleanup_removing_project_namespaces do |cluster| cluster.run_after_commit do Clusters::Cleanup::ProjectNamespaceWorker.perform_async(cluster.id) end @@ -325,7 +314,7 @@ module Clusters end def elastic_stack_adapter - application_elastic_stack || integration_elastic_stack + integration_elastic_stack end def elasticsearch_client @@ -333,11 +322,7 @@ module Clusters end def elastic_stack_available? - if application_elastic_stack_available? || integration_elastic_stack_available? - true - else - false - end + !!integration_elastic_stack_available? end def kubernetes_namespace_for(environment, deployable: environment.last_deployable) @@ -391,12 +376,8 @@ module Clusters end end - def application_prometheus_available? - integration_prometheus&.available? || application_prometheus&.available? - end - def prometheus_adapter - integration_prometheus || application_prometheus + integration_prometheus end private diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb index 125783e6ee1..162a1a3290d 100644 --- a/app/models/clusters/clusters_hierarchy.rb +++ b/app/models/clusters/clusters_hierarchy.rb @@ -4,9 +4,8 @@ module Clusters class ClustersHierarchy DEPTH_COLUMN = :depth - def initialize(clusterable, include_management_project: true) + def initialize(clusterable) @clusterable = clusterable - @include_management_project = include_management_project end # Returns clusters in order from deepest to highest group @@ -25,7 +24,7 @@ module Clusters private - attr_reader :clusterable, :include_management_project + attr_reader :clusterable def recursive_cte cte = Gitlab::SQL::RecursiveCTE.new(:clusters_cte) @@ -39,7 +38,7 @@ module Clusters raise ArgumentError, "unknown type for #{clusterable}" end - if clusterable.is_a?(::Project) && include_management_project + if clusterable.is_a?(::Project) cte << same_namespace_management_clusters_query end @@ -71,7 +70,7 @@ module Clusters # Only applicable if the clusterable is a project (most especially when # requesting project.deployment_platform). def depth_order_clause - return { DEPTH_COLUMN => :asc } unless clusterable.is_a?(::Project) && include_management_project + return { DEPTH_COLUMN => :asc } unless clusterable.is_a?(::Project) order = <<~SQL (CASE clusters.management_project_id diff --git a/app/models/commit.rb b/app/models/commit.rb index 09e43bb8f20..a1ed5eb9ab9 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -15,8 +15,6 @@ class Commit include ActsAsPaginatedDiff include CacheMarkdownField - attr_mentionable :safe_message, pipeline: :single_line - participant :author participant :committer participant :notes_with_associations @@ -35,10 +33,20 @@ class Commit # Used by GFM to match and present link extensions on node texts and hrefs. LINK_EXTENSION_PATTERN = /(patch)/.freeze + DEFAULT_MAX_DIFF_LINES_SETTING = 50_000 + DEFAULT_MAX_DIFF_FILES_SETTING = 1_000 + MAX_DIFF_LINES_SETTING_UPPER_BOUND = 100_000 + MAX_DIFF_FILES_SETTING_UPPER_BOUND = 3_000 + DIFF_SAFE_LIMIT_FACTOR = 10 + cache_markdown_field :title, pipeline: :single_line cache_markdown_field :full_title, pipeline: :single_line, limit: 1.kilobyte cache_markdown_field :description, pipeline: :commit_description, limit: 1.megabyte + # Share the cache used by the markdown fields + attr_mentionable :title, pipeline: :single_line + attr_mentionable :description, pipeline: :commit_description, limit: 1.megabyte + class << self def decorate(commits, container) commits.map do |commit| @@ -76,20 +84,24 @@ class Commit end def diff_safe_lines(project: nil) - Gitlab::Git::DiffCollection.default_limits(project: project)[:max_lines] + diff_safe_max_lines(project: project) end - def diff_hard_limit_files(project: nil) + def diff_max_files(project: nil) if Feature.enabled?(:increased_diff_limits, project) 3000 + elsif Feature.enabled?(:configurable_diff_limits, project) + Gitlab::CurrentSettings.diff_max_files else 1000 end end - def diff_hard_limit_lines(project: nil) + def diff_max_lines(project: nil) if Feature.enabled?(:increased_diff_limits, project) 100000 + elsif Feature.enabled?(:configurable_diff_limits, project) + Gitlab::CurrentSettings.diff_max_lines else 50000 end @@ -97,11 +109,19 @@ class Commit def max_diff_options(project: nil) { - max_files: diff_hard_limit_files(project: project), - max_lines: diff_hard_limit_lines(project: project) + max_files: diff_max_files(project: project), + max_lines: diff_max_lines(project: project) } end + def diff_safe_max_files(project: nil) + diff_max_files(project: project) / DIFF_SAFE_LIMIT_FACTOR + end + + def diff_safe_max_lines(project: nil) + diff_max_lines(project: project) / DIFF_SAFE_LIMIT_FACTOR + end + def from_hash(hash, container) raw_commit = Gitlab::Git::Commit.new(container.repository.raw, hash) new(raw_commit, container) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index c5ba19438cd..2db606898b9 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -57,6 +57,9 @@ class CommitStatus < ApplicationRecord scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) } scope :with_pipeline, -> { joins(:pipeline) } + scope :updated_before, ->(lookback:, timeout:) { + where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout) + } scope :for_project_paths, -> (paths) do where(project: Project.where_full_path_in(Array(paths))) @@ -174,8 +177,11 @@ class CommitStatus < ApplicationRecord next if commit_status.processed? next unless commit_status.project + last_arg = transition.args.last + transition_options = last_arg.is_a?(Hash) && last_arg.extractable_options? ? last_arg : {} + commit_status.run_after_commit do - PipelineProcessWorker.perform_async(pipeline_id) + PipelineProcessWorker.perform_async(pipeline_id) unless transition_options[:skip_pipeline_processing] ExpireJobCacheWorker.perform_async(id) end end diff --git a/app/models/concerns/bulk_insert_safe.rb b/app/models/concerns/bulk_insert_safe.rb index 3748e77e933..908f0b6a7e2 100644 --- a/app/models/concerns/bulk_insert_safe.rb +++ b/app/models/concerns/bulk_insert_safe.rb @@ -141,6 +141,12 @@ module BulkInsertSafe raise ArgumentError, "returns needs to be :ids or nil" end + # Handle insertions for tables with a composite primary key + primary_keys = connection.schema_cache.primary_keys(table_name) + if unique_by.blank? && primary_key != primary_keys + unique_by = primary_keys + end + transaction do items.each_slice(batch_size).flat_map do |item_batch| attributes = _bulk_insert_item_attributes( diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index a5cf947ba07..101bff32dfe 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -27,7 +27,7 @@ module CacheMarkdownField # Returns the default Banzai render context for the cached markdown field. def banzai_render_context(field) raise ArgumentError, "Unknown field: #{field.inspect}" unless - cached_markdown_fields.markdown_fields.include?(field) + cached_markdown_fields.key?(field) # Always include a project key, or Banzai complains project = self.project if self.respond_to?(:project) @@ -100,7 +100,7 @@ module CacheMarkdownField def cached_html_for(markdown_field) raise ArgumentError, "Unknown field: #{markdown_field}" unless - cached_markdown_fields.markdown_fields.include?(markdown_field) + cached_markdown_fields.key?(markdown_field) __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end @@ -108,7 +108,7 @@ module CacheMarkdownField # Updates the markdown cache if necessary, then returns the field # Unlike `cached_html_for` it returns `nil` if the field does not exist def updated_cached_html_for(markdown_field) - return unless cached_markdown_fields.markdown_fields.include?(markdown_field) + return unless cached_markdown_fields.key?(markdown_field) if attribute_invalidated?(cached_markdown_fields.html_field(markdown_field)) # Invalidated due to Markdown content change @@ -157,6 +157,9 @@ module CacheMarkdownField end def store_mentions! + # We can only store mentions if the mentionable is a database object + return unless self.is_a?(ApplicationRecord) + refs = all_references(self.author) references = {} diff --git a/app/models/concerns/cron_schedulable.rb b/app/models/concerns/cron_schedulable.rb index beb3a09c119..48605ecc3d7 100644 --- a/app/models/concerns/cron_schedulable.rb +++ b/app/models/concerns/cron_schedulable.rb @@ -4,23 +4,28 @@ module CronSchedulable extend ActiveSupport::Concern include Schedulable + def set_next_run_at + self.next_run_at = calculate_next_run_at + end + + private + ## # The `next_run_at` column is set to the actual execution date of worker that # triggers the schedule. This way, a schedule like `*/1 * * * *` won't be triggered # in a short interval when the worker runs irregularly by Sidekiq Memory Killer. - def set_next_run_at + def calculate_next_run_at now = Time.zone.now + ideal_next_run = ideal_next_run_from(now) - self.next_run_at = if ideal_next_run == cron_worker_next_run_from(now) - ideal_next_run - else - cron_worker_next_run_from(ideal_next_run) - end + if ideal_next_run == cron_worker_next_run_from(now) + ideal_next_run + else + cron_worker_next_run_from(ideal_next_run) + end end - private - def ideal_next_run_from(start_time) next_time_from(start_time, cron, cron_timezone) end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index 02f7711e927..b6245e29746 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -10,10 +10,6 @@ module DeploymentPlatform private - def cluster_management_project_enabled? - Feature.enabled?(:cluster_management_project, self, default_enabled: true) - end - def find_deployment_platform(environment) find_platform_kubernetes_with_cte(environment) || find_instance_cluster_platform_kubernetes(environment: environment) @@ -21,13 +17,13 @@ module DeploymentPlatform def find_platform_kubernetes_with_cte(environment) if environment - ::Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?) + ::Clusters::ClustersHierarchy.new(self) .base_and_ancestors .enabled .on_environment(environment, relevant_only: true) .first&.platform_kubernetes else - Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors + Clusters::ClustersHierarchy.new(self).base_and_ancestors .enabled.default_environment .first&.platform_kubernetes end diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb index 6d0a21cf070..c66942025d7 100644 --- a/app/models/concerns/enum_with_nil.rb +++ b/app/models/concerns/enum_with_nil.rb @@ -11,14 +11,6 @@ module EnumWithNil # override auto-defined methods only for the # key which uses nil value definitions.each do |name, values| - next unless key_with_nil = values.key(nil) - - # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } - # this overrides auto-generated method `unknown_failure?` - define_method("#{key_with_nil}?") do - self[name].nil? - end - # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } # this overrides auto-generated method `failure_reason` define_method(name) do diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb index 2e368b12cb7..72788d15c0a 100644 --- a/app/models/concerns/enums/ci/commit_status.rb +++ b/app/models/concerns/enums/ci/commit_status.rb @@ -24,6 +24,7 @@ module Enums project_deleted: 15, ci_quota_exceeded: 16, pipeline_loop_detected: 17, + no_matching_runner: 18, # not used anymore, but cannot be deleted because of old data insufficient_bridge_permissions: 1_001, downstream_bridge_project_not_found: 1_002, invalid_bridge_trigger: 1_003, diff --git a/app/models/concerns/enums/vulnerability.rb b/app/models/concerns/enums/vulnerability.rb index 55360eb92e6..749d1ad65cd 100644 --- a/app/models/concerns/enums/vulnerability.rb +++ b/app/models/concerns/enums/vulnerability.rb @@ -29,6 +29,14 @@ module Enums critical: 7 }.with_indifferent_access.freeze + DETECTION_METHODS = { + gitlab_security_report: 0, + external_security_report: 1, + bug_bounty: 2, + code_review: 3, + security_audit: 4 + }.with_indifferent_access.freeze + def self.confidence_levels CONFIDENCE_LEVELS end @@ -40,6 +48,10 @@ module Enums def self.severity_levels SEVERITY_LEVELS end + + def self.detection_methods + DETECTION_METHODS + end end end diff --git a/app/models/concerns/has_timelogs_report.rb b/app/models/concerns/has_timelogs_report.rb deleted file mode 100644 index 3af063438bf..00000000000 --- a/app/models/concerns/has_timelogs_report.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module HasTimelogsReport - extend ActiveSupport::Concern - include Gitlab::Utils::StrongMemoize - - def timelogs(start_time, end_time) - strong_memoize(:timelogs) { timelogs_for(start_time, end_time) } - end - - def user_can_access_group_timelogs?(current_user) - Ability.allowed?(current_user, :read_group_timelogs, self) - end - - private - - def timelogs_for(start_time, end_time) - Timelog.between_times(start_time, end_time).in_group(self) - end -end diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index 468387115e5..4b4f9c0df84 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -12,10 +12,11 @@ module HasUserType ghost: 5, project_bot: 6, migration_bot: 7, - security_bot: 8 + security_bot: 8, + automation_bot: 9 }.with_indifferent_access.freeze - BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot].freeze + BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot automation_bot].freeze NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze diff --git a/app/models/concerns/services/data_fields.rb b/app/models/concerns/integrations/base_data_fields.rb index fd56af449bc..3cedb90756f 100644 --- a/app/models/concerns/services/data_fields.rb +++ b/app/models/concerns/integrations/base_data_fields.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true -module Services - module DataFields +module Integrations + module BaseDataFields extend ActiveSupport::Concern included do - belongs_to :integration, inverse_of: self.name.underscore.to_sym, foreign_key: :service_id + # TODO: Once we rename the tables we can't rely on `table_name` anymore. + # https://gitlab.com/gitlab-org/gitlab/-/issues/331953 + belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: :service_id delegate :activated?, to: :integration, allow_nil: true diff --git a/app/models/concerns/integrations/has_data_fields.rb b/app/models/concerns/integrations/has_data_fields.rb new file mode 100644 index 00000000000..e9aaaac8226 --- /dev/null +++ b/app/models/concerns/integrations/has_data_fields.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Integrations + module HasDataFields + extend ActiveSupport::Concern + + class_methods do + # Provide convenient accessor methods for data fields. + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + def data_field(*args) + args.each do |arg| + self.class_eval <<-RUBY, __FILE__, __LINE__ + 1 + unless method_defined?(arg) + def #{arg} + data_fields.send('#{arg}') || (properties && properties['#{arg}']) + end + end + + def #{arg}=(value) + @old_data_fields ||= {} + @old_data_fields['#{arg}'] ||= #{arg} # set only on the first assignment, IOW we remember the original value only + data_fields.send('#{arg}=', value) + end + + def #{arg}_touched? + @old_data_fields ||= {} + @old_data_fields.has_key?('#{arg}') + end + + def #{arg}_changed? + #{arg}_touched? && @old_data_fields['#{arg}'] != #{arg} + end + + def #{arg}_was + return unless #{arg}_touched? + return if data_fields.persisted? # arg_was does not work for attr_encrypted + + legacy_properties_data['#{arg}'] + end + RUBY + end + end + end + + included do + has_one :issue_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::IssueTrackerData' + has_one :jira_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::JiraTrackerData' + has_one :open_project_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::OpenProjectTrackerData' + + def data_fields + raise NotImplementedError + end + + def data_fields_present? + data_fields.present? + rescue NotImplementedError + false + end + end + end +end diff --git a/app/models/project_services/slack_mattermost/notifier.rb b/app/models/concerns/integrations/slack_mattermost_notifier.rb index 1a78cea5933..a919fc840fd 100644 --- a/app/models/project_services/slack_mattermost/notifier.rb +++ b/app/models/concerns/integrations/slack_mattermost_notifier.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -module SlackMattermost - module Notifier +module Integrations + module SlackMattermostNotifier private def notify(message, opts) # See https://gitlab.com/gitlab-org/slack-notifier/#custom-http-client - notifier = Slack::Messenger.new(webhook, opts.merge(http_client: HTTPClient)) + notifier = ::Slack::Messenger.new(webhook, opts.merge(http_client: HTTPClient)) notifier.ping( message.pretext, attachments: message.attachments, diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index f5c70f10dc5..2d06247a486 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -101,20 +101,19 @@ module Issuable scope :unassigned, -> do where("NOT EXISTS (SELECT TRUE FROM #{to_ability_name}_assignees WHERE #{to_ability_name}_id = #{to_ability_name}s.id)") end - scope :assigned_to, ->(u) do - assignees_table = Arel::Table.new("#{to_ability_name}_assignees") - sql = assignees_table.project('true').where(assignees_table[:user_id].in(u.id)).where(Arel::Nodes::SqlLiteral.new("#{to_ability_name}_id = #{to_ability_name}s.id")) - where("EXISTS (#{sql.to_sql})") - end - # rubocop:enable GitlabSecurity/SqlInjection + scope :assigned_to, ->(users) do + assignees_class = self.reflect_on_association("#{to_ability_name}_assignees").klass + condition = assignees_class.where(user_id: users).where(Arel.sql("#{to_ability_name}_id = #{to_ability_name}s.id")) + where(condition.arel.exists) + end scope :not_assigned_to, ->(users) do - assignees_table = Arel::Table.new("#{to_ability_name}_assignees") - sql = assignees_table.project('true') - .where(assignees_table[:user_id].in(users)) - .where(Arel::Nodes::SqlLiteral.new("#{to_ability_name}_id = #{to_ability_name}s.id")) - where(sql.exists.not) + assignees_class = self.reflect_on_association("#{to_ability_name}_assignees").klass + + condition = assignees_class.where(user_id: users).where(Arel.sql("#{to_ability_name}_id = #{to_ability_name}s.id")) + where(condition.arel.exists.not) end + # rubocop:enable GitlabSecurity/SqlInjection scope :without_particular_labels, ->(label_names) do labels_table = Label.arel_table @@ -469,9 +468,11 @@ module Issuable if self.respond_to?(:total_time_spent) old_total_time_spent = old_associations.fetch(:total_time_spent, total_time_spent) + old_time_change = old_associations.fetch(:time_change, time_change) if old_total_time_spent != total_time_spent changes[:total_time_spent] = [old_total_time_spent, total_time_spent] + changes[:time_change] = [old_time_change, time_change] end end end diff --git a/app/models/concerns/issue_available_features.rb b/app/models/concerns/issue_available_features.rb index 28d12a033a6..933e8b5f687 100644 --- a/app/models/concerns/issue_available_features.rb +++ b/app/models/concerns/issue_available_features.rb @@ -11,7 +11,8 @@ module IssueAvailableFeatures def available_features_for_issue_types { assignee: %w(issue incident), - confidentiality: %(issue incident) + confidentiality: %w(issue incident), + time_tracking: %w(issue incident) }.with_indifferent_access end end diff --git a/app/models/concerns/limitable.rb b/app/models/concerns/limitable.rb index 672bcdbbb1b..41efea65c5a 100644 --- a/app/models/concerns/limitable.rb +++ b/app/models/concerns/limitable.rb @@ -6,6 +6,7 @@ module Limitable included do class_attribute :limit_scope + class_attribute :limit_relation class_attribute :limit_name class_attribute :limit_feature_flag self.limit_name = self.name.demodulize.tableize @@ -28,7 +29,7 @@ module Limitable return unless scope_relation return if limit_feature_flag && ::Feature.disabled?(limit_feature_flag, scope_relation, default_enabled: :yaml) - relation = self.class.where(limit_scope => scope_relation) + relation = limit_relation ? self.public_send(limit_relation) : self.class.where(limit_scope => scope_relation) # rubocop:disable GitlabSecurity/PublicSend limits = scope_relation.actual_limits check_plan_limit_not_exceeded(limits, relation) diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb index e33b6db0103..b05beb6c764 100644 --- a/app/models/concerns/mentionable/reference_regexes.rb +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -29,7 +29,7 @@ module Mentionable def self.external_pattern strong_memoize(:external_pattern) do - issue_pattern = IssueTrackerService.reference_pattern + issue_pattern = Integrations::BaseIssueTracker.reference_pattern link_patterns = URI::DEFAULT_PARSER.make_regexp(%w(http https)) reference_pattern(link_patterns, issue_pattern) end diff --git a/app/models/concerns/notification_branch_selection.rb b/app/models/concerns/notification_branch_selection.rb index 2354335469a..18ec996c3df 100644 --- a/app/models/concerns/notification_branch_selection.rb +++ b/app/models/concerns/notification_branch_selection.rb @@ -2,7 +2,7 @@ # Concern handling functionality around deciding whether to send notification # for activities on a specified branch or not. Will be included in -# ChatNotificationService and PipelinesEmailService classes. +# Integrations::BaseChatNotification and PipelinesEmailService classes. module NotificationBranchSelection extend ActiveSupport::Concern diff --git a/app/models/concerns/packages/debian/component_file.rb b/app/models/concerns/packages/debian/component_file.rb index c41635a0d16..9cf66c756a0 100644 --- a/app/models/concerns/packages/debian/component_file.rb +++ b/app/models/concerns/packages/debian/component_file.rb @@ -50,6 +50,8 @@ module Packages scope :with_file_type, ->(file_type) { where(file_type: file_type) } + scope :with_architecture, ->(architecture) { where(architecture: architecture) } + scope :with_architecture_name, ->(architecture_name) do left_outer_joins(:architecture) .where("packages_debian_#{container_type}_architectures" => { name: architecture_name }) @@ -60,7 +62,7 @@ module Packages scope :preload_distribution, -> { includes(component: :distribution) } - scope :created_before, ->(reference) { where("#{table_name}.created_at < ?", reference) } + scope :updated_before, ->(reference) { where("#{table_name}.updated_at < ?", reference) } mount_file_store_uploader Packages::Debian::ComponentFileUploader diff --git a/app/models/concerns/packages/debian/distribution.rb b/app/models/concerns/packages/debian/distribution.rb index 267c7a4d201..159f0044c82 100644 --- a/app/models/concerns/packages/debian/distribution.rb +++ b/app/models/concerns/packages/debian/distribution.rb @@ -18,6 +18,10 @@ module Packages belongs_to container_type belongs_to :creator, class_name: 'User' + has_one :key, + class_name: "Packages::Debian::#{container_type.capitalize}DistributionKey", + foreign_key: :distribution_id, + inverse_of: :distribution # component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :components, class_name: "Packages::Debian::#{container_type.capitalize}Component", @@ -91,6 +95,14 @@ module Packages mount_file_store_uploader Packages::Debian::DistributionReleaseFileUploader + def component_names + components.pluck(:name).sort + end + + def architecture_names + architectures.pluck(:name).sort + end + def needs_update? !file.exists? || time_duration_expired? end diff --git a/app/models/concerns/packages/debian/distribution_key.rb b/app/models/concerns/packages/debian/distribution_key.rb new file mode 100644 index 00000000000..7023e2dcd37 --- /dev/null +++ b/app/models/concerns/packages/debian/distribution_key.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Packages + module Debian + module DistributionKey + extend ActiveSupport::Concern + + included do + belongs_to :distribution, class_name: "Packages::Debian::#{container_type.capitalize}Distribution", inverse_of: :key + validates :distribution, + presence: true + + validates :private_key, presence: true, length: { maximum: 512.kilobytes } + validates :passphrase, presence: true, length: { maximum: 255 } + validates :public_key, presence: true, length: { maximum: 512.kilobytes } + validates :fingerprint, presence: true, length: { maximum: 255 } + + validate :private_key_armored, :public_key_armored + + attr_encrypted :private_key, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm' + attr_encrypted :passphrase, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm' + + private + + def private_key_armored + if private_key.present? && !private_key.start_with?('-----BEGIN PGP PRIVATE KEY BLOCK-----') + errors.add(:private_key, 'must be ASCII armored') + end + end + + def public_key_armored + if public_key.present? && !public_key.start_with?('-----BEGIN PGP PUBLIC KEY BLOCK-----') + errors.add(:public_key, 'must be ASCII armored') + end + end + end + end + end +end diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index afebc426762..86280097d19 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -38,7 +38,7 @@ module PrometheusAdapter # This is a heavy-weight check if a prometheus is properly configured and accessible from GitLab. # This actually sends a request to an external service and often it could take a long time, - # Please consider using `configured?` instead if the process is running on unicorn/puma threads. + # Please consider using `configured?` instead if the process is running on Puma threads. def can_query? prometheus_client.present? end diff --git a/app/models/concerns/service_push_data_validations.rb b/app/models/concerns/service_push_data_validations.rb index defc5794142..451804a2c56 100644 --- a/app/models/concerns/service_push_data_validations.rb +++ b/app/models/concerns/service_push_data_validations.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -# This concern is used by registerd services such as TeamCityService and -# DroneCiService and add methods to perform validations on the received +# This concern is used by registered integrations such as Integrations::TeamCity and +# Integrations::DroneCi and adds methods to perform validations on the received # data. module ServicePushDataValidations diff --git a/app/models/concerns/taggable_queries.rb b/app/models/concerns/taggable_queries.rb new file mode 100644 index 00000000000..2897e5e6420 --- /dev/null +++ b/app/models/concerns/taggable_queries.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module TaggableQueries + extend ActiveSupport::Concern + + class_methods do + # context is a name `acts_as_taggable context` + def arel_tag_names_array(context = :tags) + ActsAsTaggableOn::Tagging + .joins(:tag) + .where("taggings.taggable_id=#{quoted_table_name}.id") # rubocop:disable GitlabSecurity/SqlInjection + .where(taggings: { context: context, taggable_type: polymorphic_name }) + .select('COALESCE(array_agg(tags.name ORDER BY name), ARRAY[]::text[])') + end + end +end diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index a1e7d06b1c1..89b42eec727 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -33,11 +33,11 @@ module TimeTrackable return if @time_spent == 0 - if @time_spent == :reset - reset_spent_time - else - add_or_subtract_spent_time - end + @timelog = if @time_spent == :reset + reset_spent_time + else + add_or_subtract_spent_time + end end alias_method :spend_time=, :spend_time # rubocop:enable Gitlab/ModuleWithInstanceVariables @@ -50,6 +50,14 @@ module TimeTrackable Gitlab::TimeTrackingFormatter.output(total_time_spent) end + def time_change + @timelog&.time_spent.to_i # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def human_time_change + Gitlab::TimeTrackingFormatter.output(time_change) + end + def human_time_estimate Gitlab::TimeTrackingFormatter.output(time_estimate) end diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index fb9a8cd312d..8dc58f8dca1 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -44,7 +44,6 @@ module Timebox validates :project, presence: true, unless: :group validates :title, presence: true - validate :uniqueness_of_title, if: :title_changed? validate :timebox_type_check validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } validate :dates_within_4_digits @@ -243,18 +242,6 @@ module Timebox end end - # Timebox titles must be unique across project and group timeboxes - def uniqueness_of_title - if project - relation = self.class.for_projects_and_groups([project_id], [project.group&.id]) - elsif group - relation = self.class.for_projects_and_groups(group.projects.select(:id), [group.id]) - end - - title_exists = relation.find_by_title(title) - errors.add(:title, _("already being used for another group or project %{timebox_name}.") % { timebox_name: timebox_name }) if title_exists - end - # Timebox should be either a project timebox or a group timebox def timebox_type_check if group_id && project_id diff --git a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb index 25c050820d6..3be82ed72d3 100644 --- a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb +++ b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb @@ -5,10 +5,6 @@ module TokenAuthenticatableStrategies DYNAMIC_NONCE_IDENTIFIER = "|" NONCE_SIZE = 12 - def self.encrypt_token(plaintext_token) - Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token) - end - def self.decrypt_token(token) return unless token @@ -22,5 +18,13 @@ module TokenAuthenticatableStrategies Gitlab::CryptoHelper.aes256_gcm_decrypt(token) end end + + def self.encrypt_token(plaintext_token) + return Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token) unless Feature.enabled?(:dynamic_nonce, type: :ops) + + iv = ::Digest::SHA256.hexdigest(plaintext_token).bytes.take(NONCE_SIZE).pack('c*') + token = Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token, nonce: iv) + "#{DYNAMIC_NONCE_IDENTIFIER}#{token}#{iv}" + end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 6e0d0e347c9..2d28a81f462 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -24,8 +24,15 @@ class ContainerRepository < ApplicationRecord scope :for_group_and_its_subgroups, ->(group) do project_scope = Project .for_group_and_its_subgroups(group) - .with_container_registry - .select(:id) + + project_scope = + if Feature.enabled?(:read_container_registry_access_level, group, default_enabled: :yaml) + project_scope.with_feature_enabled(:container_registry) + else + project_scope.with_container_registry + end + + project_scope = project_scope.select(:id) joins("INNER JOIN (#{project_scope.to_sql}) projects on projects.id=container_repositories.project_id") end @@ -33,6 +40,7 @@ class ContainerRepository < ApplicationRecord scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :waiting_for_cleanup, -> { where(expiration_policy_cleanup_status: WAITING_CLEANUP_STATUSES) } scope :expiration_policy_started_at_nil_or_before, ->(timestamp) { where('expiration_policy_started_at < ? OR expiration_policy_started_at IS NULL', timestamp) } + scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.where('expiration_policy_started_at < ?', threshold) } def self.exists_by_path?(path) where( @@ -42,16 +50,15 @@ class ContainerRepository < ApplicationRecord end def self.with_enabled_policy - joins("INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id") + joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id') .where(container_expiration_policies: { enabled: true }) end def self.requiring_cleanup - where( - container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES }, - project_id: ::ContainerExpirationPolicy.runnable_schedules - .select(:project_id) - ) + with_enabled_policy + .where(container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES }) + .where('container_repositories.expiration_policy_started_at IS NULL OR container_repositories.expiration_policy_started_at < container_expiration_policies.next_run_at') + .where('container_expiration_policies.next_run_at < ?', Time.zone.now) end def self.with_unfinished_cleanup diff --git a/app/models/cycle_analytics/project_level.rb b/app/models/cycle_analytics/project_level.rb deleted file mode 100644 index 5bd07b3f6c3..00000000000 --- a/app/models/cycle_analytics/project_level.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module CycleAnalytics - class ProjectLevel - attr_reader :project, :options - - def initialize(project, options:) - @project = project - @options = options.merge(project: project) - end - - def summary - @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new(project, - from: options[:from], - to: options[:to], - current_user: options[:current_user]).data - end - - def permissions(user:) - Gitlab::CycleAnalytics::Permissions.get(user: user, project: project) - end - - def stats - @stats ||= default_stage_names.map do |stage_name| - self[stage_name].as_json - end - end - - def [](stage_name) - CycleAnalytics::ProjectLevelStageAdapter.new(build_stage(stage_name), options) - end - - private - - def build_stage(stage_name) - stage_params = stage_params_by_name(stage_name).merge(project: project) - Analytics::CycleAnalytics::ProjectStage.new(stage_params) - end - - def stage_params_by_name(name) - Gitlab::Analytics::CycleAnalytics::DefaultStages.find_by_name!(name) - end - - def default_stage_names - Gitlab::Analytics::CycleAnalytics::DefaultStages.symbolized_stage_names - end - end -end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index e2b25690323..7f5849bffc6 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -8,6 +8,9 @@ class Deployment < ApplicationRecord include Importable include Gitlab::Utils::StrongMemoize include FastDestroyAll + include IgnorableColumns + + ignore_column :deployable_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/environment.rb b/app/models/environment.rb index 2e677a3d177..558963c98c4 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -223,6 +223,7 @@ class Environment < ApplicationRecord Gitlab::Ci::Variables::Collection.new .append(key: 'CI_ENVIRONMENT_NAME', value: name) .append(key: 'CI_ENVIRONMENT_SLUG', value: slug) + .append(key: 'CI_ENVIRONMENT_TIER', value: tier) end def recently_updated_on_branch?(ref) @@ -335,10 +336,6 @@ class Environment < ApplicationRecord prometheus_adapter.query(:environment, self) if has_metrics_and_can_query? end - def prometheus_status - deployment_platform&.cluster&.application_prometheus&.status_name - end - def additional_metrics(*args) return unless has_metrics_and_can_query? diff --git a/app/models/environment_status.rb b/app/models/environment_status.rb index 55ea4e2fe18..07c0983f239 100644 --- a/app/models/environment_status.rb +++ b/app/models/environment_status.rb @@ -100,7 +100,7 @@ class EnvironmentStatus def self.build_environments_status(mr, user, pipeline) return [] unless pipeline - pipeline.environments.includes(:project).available.map do |environment| + pipeline.environments_in_self_and_descendants.includes(:project).available.map do |environment| next unless Ability.allowed?(user, :read_environment, environment) EnvironmentStatus.new(pipeline.project, environment, mr, pipeline.sha) diff --git a/app/models/experiment.rb b/app/models/experiment.rb index 7ffb321f2b7..cd0814c476a 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -11,7 +11,11 @@ class Experiment < ApplicationRecord end def self.add_group(name, variant:, group:) - find_or_create_by!(name: name).record_group_and_variant!(group, variant) + add_subject(name, variant: variant, subject: group) + end + + def self.add_subject(name, variant:, subject:) + find_or_create_by!(name: name).record_subject_and_variant!(subject, variant) end def self.record_conversion_event(name, user, context = {}) @@ -37,8 +41,11 @@ class Experiment < ApplicationRecord experiment_user.update!(converted_at: Time.current, context: merged_context(experiment_user, context)) end - def record_group_and_variant!(group, variant) - experiment_subject = experiment_subjects.find_or_initialize_by(group: group) + def record_subject_and_variant!(subject, variant) + raise 'Incompatible subject provided!' unless ExperimentSubject.valid_subject?(subject) + + attr_name = subject.class.table_name.singularize.to_sym + experiment_subject = experiment_subjects.find_or_initialize_by(attr_name => subject) experiment_subject.assign_attributes(variant: variant) # We only call save when necessary because this causes the request to stick to the primary DB # even when the save is a no-op diff --git a/app/models/experiment_subject.rb b/app/models/experiment_subject.rb index 51ffc0b304e..2a7b9017a51 100644 --- a/app/models/experiment_subject.rb +++ b/app/models/experiment_subject.rb @@ -5,7 +5,7 @@ class ExperimentSubject < ApplicationRecord belongs_to :experiment, inverse_of: :experiment_subjects belongs_to :user - belongs_to :group + belongs_to :namespace belongs_to :project validates :experiment, presence: true @@ -14,15 +14,19 @@ class ExperimentSubject < ApplicationRecord enum variant: { GROUP_CONTROL => 0, GROUP_EXPERIMENTAL => 1 } + def self.valid_subject?(subject) + subject.is_a?(Namespace) || subject.is_a?(User) || subject.is_a?(Project) + end + private def must_have_one_subject_present if non_nil_subjects.length != 1 - errors.add(:base, s_("ExperimentSubject|Must have exactly one of User, Group, or Project.")) + errors.add(:base, s_("ExperimentSubject|Must have exactly one of User, Namespace, or Project.")) end end def non_nil_subjects - @non_nil_subjects ||= [user, group, project].reject(&:blank?) + @non_nil_subjects ||= [user, namespace, project].reject(&:blank?) end end diff --git a/app/models/group.rb b/app/models/group.rb index da795651c63..e4127b2b2d4 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -16,9 +16,7 @@ class Group < Namespace include Gitlab::Utils::StrongMemoize include GroupAPICompatibility include EachBatch - include HasTimelogsReport - - ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 + include BulkMemberAccessLoad has_many :all_group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent has_many :group_members, -> { where(requested_at: nil).where.not(members: { access_level: Gitlab::Access::MINIMAL_ACCESS }) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent @@ -82,6 +80,8 @@ class Group < Namespace # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :debian_distributions, class_name: 'Packages::Debian::GroupDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + delegate :prevent_sharing_groups_outside_hierarchy, to: :namespace_settings + accepts_nested_attributes_for :variables, allow_destroy: true validate :visibility_level_allowed_by_projects @@ -444,6 +444,12 @@ class Group < Namespace end end + def self_and_descendants_ids + strong_memoize(:self_and_descendants_ids) do + self_and_descendants.pluck(:id) + end + end + def direct_members GroupMember.active_without_invites_and_requests .non_minimal_access @@ -569,24 +575,8 @@ class Group < Namespace def max_member_access_for_user(user, only_concrete_membership: false) return GroupMember::NO_ACCESS unless user return GroupMember::OWNER if user.can_admin_all_resources? && !only_concrete_membership - # Use the preloaded value that exists instead of performing the db query again(cached or not). - # Groups::GroupMembersController#preload_max_access makes use of this by - # calling Group#max_member_access. This helps when we have a process - # that may query this multiple times from the outside through a policy query - # like the GroupPolicy#lookup_access_level! does as a condition for any role - return user.max_access_for_group[id] if user.max_access_for_group[id] - max_member_access(user) - end - - def max_member_access(user) - max_member_access = members_with_parents - .where(user_id: user) - .reorder(access_level: :desc) - .first - &.access_level - - max_member_access || GroupMember::NO_ACCESS + max_member_access([user.id])[user.id] end def mattermost_team_params @@ -649,7 +639,7 @@ class Group < Namespace end def access_request_approvers_to_be_notified - members.owners.connected_to_user.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) + members.owners.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end def supports_events? @@ -657,13 +647,17 @@ class Group < Namespace end def export_file_exists? - export_file&.file + import_export_upload&.export_file_exists? end def export_file import_export_upload&.export_file end + def export_archive_exists? + import_export_upload&.export_archive_exists? + end + def adjourned_deletion? false end @@ -728,8 +722,26 @@ class Group < Namespace Gitlab::Routing.url_helpers.activity_group_path(self) end + # rubocop: disable CodeReuse/ServiceClass + def open_issues_count(current_user = nil) + Groups::OpenIssuesCountService.new(self, current_user).count + end + # rubocop: enable CodeReuse/ServiceClass + + # rubocop: disable CodeReuse/ServiceClass + def open_merge_requests_count(current_user = nil) + Groups::MergeRequestsCountService.new(self, current_user).count + end + # rubocop: enable CodeReuse/ServiceClass + private + def max_member_access(user_ids) + max_member_access_for_resource_ids(User, user_ids) do |user_ids| + members_with_parents.where(user_id: user_ids).group(:user_id).maximum(:access_level) + end + end + def update_two_factor_requirement return unless saved_change_to_require_two_factor_authentication? || saved_change_to_two_factor_grace_period? diff --git a/app/models/group_deploy_token.rb b/app/models/group_deploy_token.rb index d4ad29ddabb..084a8672460 100644 --- a/app/models/group_deploy_token.rb +++ b/app/models/group_deploy_token.rb @@ -9,8 +9,6 @@ class GroupDeployToken < ApplicationRecord validates :deploy_token_id, uniqueness: { scope: [:group_id] } def has_access_to?(requested_project) - return false unless Feature.enabled?(:allow_group_deploy_token, default_enabled: true) - requested_project_group = requested_project&.group return false unless requested_project_group return true if requested_project_group.id == group_id diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index a28b97e63e5..d1584a62bfb 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -39,6 +39,11 @@ class ProjectHook < WebHook def rate_limit project.actual_limits.limit_for(:web_hook_calls) end + + override :application_context + def application_context + super.merge(project: project) + end end ProjectHook.prepend_mod_with('ProjectHook') diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 02b4feb4ccc..5f8fa4bca0a 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -3,6 +3,7 @@ class WebHook < ApplicationRecord include Sortable + MAX_FAILURES = 100 FAILURE_THRESHOLD = 3 # three strikes INITIAL_BACKOFF = 10.minutes MAX_BACKOFF = 1.day @@ -72,14 +73,29 @@ class WebHook < ApplicationRecord end def enable! + return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 + update!(recent_failures: 0, disabled_until: nil, backoff_count: 0) end + def backoff! + update!(disabled_until: next_backoff.from_now, backoff_count: backoff_count.succ.clamp(0, MAX_FAILURES)) + end + + def failed! + update!(recent_failures: recent_failures + 1) if recent_failures < MAX_FAILURES + end + # Overridden in ProjectHook and GroupHook, other webhooks are not rate-limited. def rate_limit nil end + # Custom attributes to be included in the worker context. + def application_context + { related_class: type } + end + private def web_hooks_disable_failed? diff --git a/app/models/hooks/web_hook_log_archived.rb b/app/models/hooks/web_hook_log_archived.rb deleted file mode 100644 index a1c8a44f5ba..00000000000 --- a/app/models/hooks/web_hook_log_archived.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -# This model is not intended to be used. -# It is a temporary reference to the old non-partitioned -# web_hook_logs table. -# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/5558 -# for details. -# rubocop:disable Gitlab/NamespacedClass: This is a temporary class with no relevant namespace -# WebHook, WebHookLog and all hooks are defined outside of a namespace -class WebHookLogArchived < ApplicationRecord - self.table_name = 'web_hook_logs_archived' -end diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb index 7d73fd281f1..bc363cce8dd 100644 --- a/app/models/import_export_upload.rb +++ b/app/models/import_export_upload.rb @@ -11,7 +11,42 @@ class ImportExportUpload < ApplicationRecord mount_uploader :import_file, ImportExportUploader mount_uploader :export_file, ImportExportUploader + # This causes CarrierWave v1 and v3 (but not v2) to upload the file to + # object storage *after* the database entry has been committed to the + # database. This avoids idling in a transaction. + if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_STORE_EXPORT_FILE_AFTER_COMMIT', true)) + skip_callback :save, :after, :store_export_file! + set_callback :commit, :after, :store_export_file! + end + + scope :updated_before, ->(date) { where('updated_at < ?', date) } + scope :with_export_file, -> { where.not(export_file: nil) } + def retrieve_upload(_identifier, paths) Upload.find_by(model: self, path: paths) end + + def export_file_exists? + !!carrierwave_export_file + end + + # This checks if the export archive is actually stored on disk. It + # requires a HEAD request if object storage is used. + def export_archive_exists? + !!carrierwave_export_file&.exists? + # Handle any HTTP unexpected error + # https://github.com/excon/excon/blob/bbb5bd791d0bb2251593b80e3bce98dbec6e8f24/lib/excon/error.rb#L129-L169 + rescue Excon::Error => e + # The HEAD request will fail with a 403 Forbidden if the file does not + # exist, and the user does not have permission to list the object + # storage bucket. + Gitlab::ErrorTracking.track_exception(e) + false + end + + private + + def carrierwave_export_file + export_file&.file + end end diff --git a/app/models/integration.rb b/app/models/integration.rb index 13203cd4e95..238ecbbf209 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -6,7 +6,7 @@ class Integration < ApplicationRecord include Sortable include Importable include ProjectServicesLoggable - include DataFields + include Integrations::HasDataFields include FromUnion include EachBatch @@ -29,6 +29,27 @@ class Integration < ApplicationRecord mock_ci mock_monitoring ].freeze + # Base classes which aren't actual integrations. + BASE_CLASSES = %w[ + Integrations::BaseChatNotification + Integrations::BaseCi + Integrations::BaseIssueTracker + Integrations::BaseMonitoring + Integrations::BaseSlashCommands + ].freeze + + # used as part of the renaming effort (https://gitlab.com/groups/gitlab-org/-/epics/2504) + RENAMED_TO_INTEGRATION = %w[ + asana assembla + bamboo bugzilla buildkite + campfire confluence custom_issue_tracker + datadog discord drone_ci + ].to_set.freeze + + def self.renamed?(name) + RENAMED_TO_INTEGRATION.include?(name) + end + serialize :properties, JSON # rubocop:disable Cop/ActiveRecordSerialize attribute :type, Gitlab::Integrations::StiType.new @@ -59,7 +80,7 @@ class Integration < ApplicationRecord validates :project_id, presence: true, unless: -> { template? || instance_level? || group_level? } validates :group_id, presence: true, unless: -> { template? || instance_level? || project_level? } validates :project_id, :group_id, absence: true, if: -> { template? || instance_level? } - validates :type, presence: true + validates :type, presence: true, exclusion: BASE_CLASSES validates :type, uniqueness: { scope: :template }, if: :template? validates :type, uniqueness: { scope: :instance }, if: :instance_level? validates :type, uniqueness: { scope: :project_id }, if: :project_level? @@ -185,7 +206,7 @@ class Integration < ApplicationRecord def self.find_or_initialize_non_project_specific_integration(name, instance: false, group_id: nil) return unless name.in?(available_services_names(include_project_specific: false)) - service_name_to_model(name).find_or_initialize_by(instance: instance, group_id: group_id) + integration_name_to_model(name).find_or_initialize_by(instance: instance, group_id: group_id) end def self.find_or_initialize_all_non_project_specific(scope) @@ -194,7 +215,7 @@ class Integration < ApplicationRecord def self.build_nonexistent_services_for(scope) nonexistent_services_types_for(scope).map do |service_type| - service_type_to_model(service_type).new + integration_type_to_model(service_type).new end end private_class_method :build_nonexistent_services_for @@ -210,6 +231,7 @@ class Integration < ApplicationRecord # Returns a list of available service names. # Example: ["asana", ...] + # @deprecated def self.available_services_names(include_project_specific: true, include_dev: true) service_names = services_names service_names += project_specific_services_names if include_project_specific @@ -218,10 +240,14 @@ class Integration < ApplicationRecord service_names.sort_by(&:downcase) end - def self.services_names + def self.integration_names INTEGRATION_NAMES end + def self.services_names + integration_names + end + def self.dev_services_names return [] unless Rails.env.development? @@ -236,29 +262,29 @@ class Integration < ApplicationRecord # Example: ["AsanaService", ...] def self.available_services_types(include_project_specific: true, include_dev: true) available_services_names(include_project_specific: include_project_specific, include_dev: include_dev).map do |service_name| - service_name_to_type(service_name) + integration_name_to_type(service_name) end end # Returns the model for the given service name. # Example: "asana" => Integrations::Asana - def self.service_name_to_model(name) - type = service_name_to_type(name) - service_type_to_model(type) + def self.integration_name_to_model(name) + type = integration_name_to_type(name) + integration_type_to_model(type) end # Returns the STI type for the given service name. # Example: "asana" => "AsanaService" - def self.service_name_to_type(name) + def self.integration_name_to_type(name) "#{name}_service".camelize end # Returns the model for the given STI type. # Example: "AsanaService" => Integrations::Asana - def self.service_type_to_model(type) + def self.integration_type_to_model(type) Gitlab::Integrations::StiType.new.cast(type).constantize end - private_class_method :service_type_to_model + private_class_method :integration_type_to_model def self.build_from_integration(integration, project_id: nil, group_id: nil) new_integration = integration.dup @@ -480,10 +506,6 @@ class Integration < ApplicationRecord ProjectServiceWorker.perform_async(id, data) end - def external_wiki? - type == 'ExternalWikiService' && active? - end - # override if needed def supports_data_fields? false diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 82111c7322e..dbd7aedf4fe 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Integrations - class Bamboo < CiService + class Bamboo < BaseCi include ActionView::Helpers::UrlHelper include ReactiveService diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb new file mode 100644 index 00000000000..5eae8bce92a --- /dev/null +++ b/app/models/integrations/base_chat_notification.rb @@ -0,0 +1,255 @@ +# frozen_string_literal: true + +# Base class for Chat notifications services +# This class is not meant to be used directly, but only to inherit from. + +module Integrations + class BaseChatNotification < Integration + include ChatMessage + include NotificationBranchSelection + + SUPPORTED_EVENTS = %w[ + push issue confidential_issue merge_request note confidential_note + tag_push pipeline wiki_page deployment + ].freeze + + SUPPORTED_EVENTS_FOR_LABEL_FILTER = %w[issue confidential_issue merge_request note confidential_note].freeze + + EVENT_CHANNEL = proc { |event| "#{event}_channel" } + + LABEL_NOTIFICATION_BEHAVIOURS = [ + MATCH_ANY_LABEL = 'match_any', + MATCH_ALL_LABELS = 'match_all' + ].freeze + + default_value_for :category, 'chat' + + prop_accessor :webhook, :username, :channel, :branches_to_be_notified, :labels_to_be_notified, :labels_to_be_notified_behavior + + # Custom serialized properties initialization + prop_accessor(*SUPPORTED_EVENTS.map { |event| EVENT_CHANNEL[event] }) + + boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch + + validates :webhook, presence: true, public_url: true, if: :activated? + validates :labels_to_be_notified_behavior, inclusion: { in: LABEL_NOTIFICATION_BEHAVIOURS }, allow_blank: true + + def initialize_properties + if properties.nil? + self.properties = {} + self.notify_only_broken_pipelines = true + self.branches_to_be_notified = "default" + self.labels_to_be_notified_behavior = MATCH_ANY_LABEL + elsif !self.notify_only_default_branch.nil? + # In older versions, there was only a boolean property named + # `notify_only_default_branch`. Now we have a string property named + # `branches_to_be_notified`. Instead of doing a background migration, we + # opted to set a value for the new property based on the old one, if + # users haven't specified one already. When users edit the service and + # select a value for this new property, it will override everything. + + self.branches_to_be_notified ||= notify_only_default_branch? ? "default" : "all" + end + end + + def confidential_issue_channel + properties['confidential_issue_channel'].presence || properties['issue_channel'] + end + + def confidential_note_channel + properties['confidential_note_channel'].presence || properties['note_channel'] + end + + def self.supported_events + SUPPORTED_EVENTS + end + + def fields + default_fields + build_event_channels + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}", required: true }.freeze, + { type: 'text', name: 'username', placeholder: 'GitLab-integration' }.freeze, + { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'Do not send notifications for successful pipelines.' }.freeze, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }.freeze, + { + type: 'text', + name: 'labels_to_be_notified', + placeholder: '~backend,~frontend', + help: 'Send notifications for issue, merge request, and comment events with the listed labels only. Leave blank to receive notifications for all events.' + }.freeze, + { + type: 'select', + name: 'labels_to_be_notified_behavior', + choices: [ + ['Match any of the labels', MATCH_ANY_LABEL], + ['Match all of the labels', MATCH_ALL_LABELS] + ] + }.freeze + ].freeze + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + return unless webhook.present? + + object_kind = data[:object_kind] + + data = custom_data(data) + + return unless notify_label?(data) + + # WebHook events often have an 'update' event that follows a 'open' or + # 'close' action. Ignore update events for now to prevent duplicate + # messages from arriving. + + message = get_message(object_kind, data) + + return false unless message + + event_type = data[:event_type] || object_kind + + channel_names = get_channel_field(event_type).presence || channel.presence + channels = channel_names&.split(',')&.map(&:strip) + + opts = {} + opts[:channel] = channels if channels.present? + opts[:username] = username if username + + if notify(message, opts) + log_usage(event_type, user_id_from_hook_data(data)) + return true + end + + false + end + + def event_channel_names + supported_events.map { |event| event_channel_name(event) } + end + + def event_field(event) + fields.find { |field| field[:name] == event_channel_name(event) } + end + + def global_fields + fields.reject { |field| field[:name].end_with?('channel') } + end + + def default_channel_placeholder + raise NotImplementedError + end + + private + + def log_usage(_, _) + # Implement in child class + end + + def labels_to_be_notified_list + return [] if labels_to_be_notified.nil? + + labels_to_be_notified.delete('~').split(',').map(&:strip) + end + + def notify_label?(data) + return true unless SUPPORTED_EVENTS_FOR_LABEL_FILTER.include?(data[:object_kind]) && labels_to_be_notified.present? + + labels = data[:labels] || data.dig(:issue, :labels) || data.dig(:merge_request, :labels) || data.dig(:object_attributes, :labels) + + return false if labels.blank? + + matching_labels = labels_to_be_notified_list & labels.pluck(:title) + + if labels_to_be_notified_behavior == MATCH_ALL_LABELS + labels_to_be_notified_list.difference(matching_labels).empty? + else + matching_labels.any? + end + end + + def user_id_from_hook_data(data) + data.dig(:user, :id) || data[:user_id] + end + + # every notifier must implement this independently + def notify(message, opts) + raise NotImplementedError + end + + def custom_data(data) + data.merge(project_url: project_url, project_name: project_name).with_indifferent_access + end + + def get_message(object_kind, data) + case object_kind + when "push", "tag_push" + Integrations::ChatMessage::PushMessage.new(data) if notify_for_ref?(data) + when "issue" + Integrations::ChatMessage::IssueMessage.new(data) unless update?(data) + when "merge_request" + Integrations::ChatMessage::MergeMessage.new(data) unless update?(data) + when "note" + Integrations::ChatMessage::NoteMessage.new(data) + when "pipeline" + Integrations::ChatMessage::PipelineMessage.new(data) if should_pipeline_be_notified?(data) + when "wiki_page" + Integrations::ChatMessage::WikiPageMessage.new(data) + when "deployment" + Integrations::ChatMessage::DeploymentMessage.new(data) + end + end + + def get_channel_field(event) + field_name = event_channel_name(event) + self.public_send(field_name) # rubocop:disable GitlabSecurity/PublicSend + end + + def build_event_channels + supported_events.reduce([]) do |channels, event| + channels << { type: 'text', name: event_channel_name(event), placeholder: default_channel_placeholder } + end + end + + def event_channel_name(event) + EVENT_CHANNEL[event] + end + + def project_name + project.full_name + end + + def project_url + project.web_url + end + + def update?(data) + data[:object_attributes][:action] == 'update' + end + + def should_pipeline_be_notified?(data) + notify_for_ref?(data) && notify_for_pipeline?(data) + end + + def notify_for_ref?(data) + return true if data[:object_kind] == 'tag_push' + return true if data.dig(:object_attributes, :tag) + + notify_for_branch?(data) + end + + def notify_for_pipeline?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end + end +end diff --git a/app/models/integrations/base_ci.rb b/app/models/integrations/base_ci.rb new file mode 100644 index 00000000000..b2e269b1b50 --- /dev/null +++ b/app/models/integrations/base_ci.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Base class for CI services +# List methods you need to implement to get your CI service +# working with GitLab merge requests +module Integrations + class BaseCi < Integration + default_value_for :category, 'ci' + + def valid_token?(token) + self.respond_to?(:token) && self.token.present? && ActiveSupport::SecurityUtils.secure_compare(token, self.token) + end + + def self.supported_events + %w(push) + end + + # Return complete url to build page + # + # Ex. + # http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c + # + def build_page(sha, ref) + # implement inside child + end + + # Return string with build status or :error symbol + # + # Allowed states: 'success', 'failed', 'running', 'pending', 'skipped' + # + # + # Ex. + # @service.commit_status('13be4ac', 'master') + # # => 'success' + # + # @service.commit_status('2abe4ac', 'dev') + # # => 'running' + # + # + def commit_status(sha, ref) + # implement inside child + end + end +end diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb new file mode 100644 index 00000000000..6c24f762cd5 --- /dev/null +++ b/app/models/integrations/base_issue_tracker.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Integrations + class BaseIssueTracker < Integration + validate :one_issue_tracker, if: :activated?, on: :manual_change + + # TODO: we can probably just delegate as part of + # https://gitlab.com/gitlab-org/gitlab/issues/29404 + data_field :project_url, :issues_url, :new_issue_url + + default_value_for :category, 'issue_tracker' + + before_validation :handle_properties + before_validation :set_default_data, on: :create + + # Pattern used to extract links from comments + # Override this method on services that uses different patterns + # This pattern does not support cross-project references + # The other code assumes that this pattern is a superset of all + # overridden patterns. See ReferenceRegexes.external_pattern + def self.reference_pattern(only_long: false) + if only_long + /(\b[A-Z][A-Z0-9_]*-)#{Gitlab::Regex.issue}/ + else + /(\b[A-Z][A-Z0-9_]*-|#{Issue.reference_prefix})#{Gitlab::Regex.issue}/ + end + end + + def handle_properties + # this has been moved from initialize_properties and should be improved + # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + return unless properties + + @legacy_properties_data = properties.dup + data_values = properties.slice!('title', 'description') + data_values.reject! { |key| data_fields.changed.include?(key) } + data_values.slice!(*data_fields.attributes.keys) + data_fields.assign_attributes(data_values) if data_values.present? + + self.properties = {} + end + + def legacy_properties_data + @legacy_properties_data ||= {} + end + + def supports_data_fields? + true + end + + def data_fields + issue_tracker_data || self.build_issue_tracker_data + end + + def default? + default + end + + def issue_url(iid) + issues_url.gsub(':id', iid.to_s) + end + + def issue_tracker_path + project_url + end + + def new_issue_path + new_issue_url + end + + def issue_path(iid) + issue_url(iid) + end + + def fields + [ + { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in the external issue tracker.'), required: true }, + { type: 'text', name: 'issues_url', title: s_('IssueTracker|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true }, + { type: 'text', name: 'new_issue_url', title: s_('IssueTracker|New issue URL'), help: s_('IssueTracker|The URL to create an issue in the external issue tracker.'), required: true } + ] + end + + def initialize_properties + {} + end + + # Initialize with default properties values + def set_default_data + return unless issues_tracker.present? + + # we don't want to override if we have set something + return if project_url || issues_url || new_issue_url + + data_fields.project_url = issues_tracker['project_url'] + data_fields.issues_url = issues_tracker['issues_url'] + data_fields.new_issue_url = issues_tracker['new_issue_url'] + end + + def self.supported_events + %w(push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + message = "#{self.type} was unable to reach #{self.project_url}. Check the url and try again." + result = false + + begin + response = Gitlab::HTTP.head(self.project_url, verify: true) + + if response + message = "#{self.type} received response #{response.code} when attempting to connect to #{self.project_url}" + result = true + end + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => error + message = "#{self.type} had an error when trying to connect to #{self.project_url}: #{error.message}" + end + log_info(message) + result + end + + def support_close_issue? + false + end + + def support_cross_reference? + false + end + + def create_cross_reference_note(mentioned, noteable, author) + # implement inside child + end + + private + + def enabled_in_gitlab_config + Gitlab.config.issues_tracker && + Gitlab.config.issues_tracker.values.any? && + issues_tracker + end + + def issues_tracker + Gitlab.config.issues_tracker[to_param] + end + + def one_issue_tracker + return if template? || instance? + return if project.blank? + + if project.integrations.external_issue_trackers.where.not(id: id).any? + errors.add(:base, _('Another issue tracker is already in use. Only one issue tracker service can be active at a time')) + end + end + end +end diff --git a/app/models/integrations/base_slash_commands.rb b/app/models/integrations/base_slash_commands.rb new file mode 100644 index 00000000000..eacf1184aae --- /dev/null +++ b/app/models/integrations/base_slash_commands.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Base class for ChatOps integrations +# This class is not meant to be used directly, but only to inherrit from. +module Integrations + class BaseSlashCommands < Integration + default_value_for :category, 'chat' + + prop_accessor :token + + has_many :chat_names, foreign_key: :service_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + + def valid_token?(token) + self.respond_to?(:token) && + self.token.present? && + ActiveSupport::SecurityUtils.secure_compare(token, self.token) + end + + def self.supported_events + %w() + end + + def can_test? + false + end + + def fields + [ + { type: 'text', name: 'token', placeholder: 'XXxxXXxxXXxxXXxxXXxxXXxx' } + ] + end + + def trigger(params) + return unless valid_token?(params[:token]) + + chat_user = find_chat_user(params) + user = chat_user&.user + + if user + unless user.can?(:use_slash_commands) + return Gitlab::SlashCommands::Presenters::Access.new.deactivated if user.deactivated? + + return Gitlab::SlashCommands::Presenters::Access.new.access_denied(project) + end + + Gitlab::SlashCommands::Command.new(project, chat_user, params).execute + else + url = authorize_chat_name_url(params) + Gitlab::SlashCommands::Presenters::Access.new(url).authorize + end + end + + private + + # rubocop: disable CodeReuse/ServiceClass + def find_chat_user(params) + ChatNames::FindUserService.new(self, params).execute + end + # rubocop: enable CodeReuse/ServiceClass + + # rubocop: disable CodeReuse/ServiceClass + def authorize_chat_name_url(params) + ChatNames::AuthorizeUserService.new(self, params).execute + end + # rubocop: enable CodeReuse/ServiceClass + end +end diff --git a/app/models/integrations/bugzilla.rb b/app/models/integrations/bugzilla.rb new file mode 100644 index 00000000000..9251015acb8 --- /dev/null +++ b/app/models/integrations/bugzilla.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Integrations + class Bugzilla < BaseIssueTracker + include ActionView::Helpers::UrlHelper + + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? + + def title + 'Bugzilla' + end + + def description + s_("IssueTracker|Use Bugzilla as this project's issue tracker.") + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bugzilla'), target: '_blank', rel: 'noopener noreferrer' + s_("IssueTracker|Use Bugzilla as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'bugzilla' + end + end +end diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb new file mode 100644 index 00000000000..906a5d02f9c --- /dev/null +++ b/app/models/integrations/buildkite.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require "addressable/uri" + +module Integrations + class Buildkite < BaseCi + include ReactiveService + + ENDPOINT = "https://buildkite.com" + + prop_accessor :project_url, :token + + validates :project_url, presence: true, public_url: true, if: :activated? + validates :token, presence: true, if: :activated? + + after_save :compose_service_hook, if: :activated? + + def self.supported_events + %w(push merge_request tag_push) + end + + # This is a stub method to work with deprecated API response + # TODO: remove enable_ssl_verification after 14.0 + # https://gitlab.com/gitlab-org/gitlab/-/issues/222808 + def enable_ssl_verification + true + end + + # Since SSL verification will always be enabled for Buildkite, + # we no longer needs to store the boolean. + # This is a stub method to work with deprecated API param. + # TODO: remove enable_ssl_verification after 14.0 + # https://gitlab.com/gitlab-org/gitlab/-/issues/222808 + def enable_ssl_verification=(_value) + self.properties.delete('enable_ssl_verification') # Remove unused key + end + + def webhook_url + "#{buildkite_endpoint('webhook')}/deliver/#{webhook_token}" + end + + def compose_service_hook + hook = service_hook || build_service_hook + hook.url = webhook_url + hook.enable_ssl_verification = true + hook.save + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + service_hook.execute(data) + end + + def commit_status(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + end + + def commit_status_path(sha) + "#{buildkite_endpoint('gitlab')}/status/#{status_token}.json?commit=#{sha}" + end + + def build_page(sha, ref) + "#{project_url}/builds?commit=#{sha}" + end + + def title + 'Buildkite' + end + + def description + 'Run CI/CD pipelines with Buildkite.' + end + + def self.to_param + 'buildkite' + end + + def fields + [ + { type: 'text', + name: 'token', + title: 'Integration Token', + help: 'This token will be provided when you create a Buildkite pipeline with a GitLab repository', + required: true }, + + { type: 'text', + name: 'project_url', + title: 'Pipeline URL', + placeholder: "#{ENDPOINT}/acme-inc/test-pipeline", + required: true } + ] + end + + def calculate_reactive_cache(sha, ref) + response = Gitlab::HTTP.try_get(commit_status_path(sha), request_options) + + status = + if response&.code == 200 && response['status'] + response['status'] + else + :error + end + + { commit_status: status } + end + + private + + def webhook_token + token_parts.first + end + + def status_token + token_parts.second + end + + def token_parts + if token.present? + token.split(':') + else + [] + end + end + + def buildkite_endpoint(subdomain = nil) + if subdomain.present? + uri = Addressable::URI.parse(ENDPOINT) + new_endpoint = "#{uri.scheme || 'http'}://#{subdomain}.#{uri.host}" + + if uri.port.present? + "#{new_endpoint}:#{uri.port}" + else + new_endpoint + end + else + ENDPOINT + end + end + + def request_options + { verify: false, extra_log_info: { project_id: project_id } } + end + end +end diff --git a/app/models/integrations/builds_email.rb b/app/models/integrations/builds_email.rb deleted file mode 100644 index 2628848667e..00000000000 --- a/app/models/integrations/builds_email.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -# This class is to be removed with 9.1 -# We should also by then remove BuildsEmailService from database -# https://gitlab.com/gitlab-org/gitlab/-/issues/331064 -module Integrations - class BuildsEmail < Integration - def self.to_param - 'builds_email' - end - - def self.supported_events - %w[] - end - end -end diff --git a/app/models/integrations/chat_message/base_message.rb b/app/models/integrations/chat_message/base_message.rb index 2f70384d3b9..afe3ffc45a0 100644 --- a/app/models/integrations/chat_message/base_message.rb +++ b/app/models/integrations/chat_message/base_message.rb @@ -58,7 +58,7 @@ module Integrations end def format(string) - Slack::Messenger::Util::LinkFormatter.format(format_relative_links(string)) + ::Slack::Messenger::Util::LinkFormatter.format(format_relative_links(string)) end def format_relative_links(string) diff --git a/app/models/integrations/chat_message/pipeline_message.rb b/app/models/integrations/chat_message/pipeline_message.rb index a0f6f582e4c..a3f68d34035 100644 --- a/app/models/integrations/chat_message/pipeline_message.rb +++ b/app/models/integrations/chat_message/pipeline_message.rb @@ -105,7 +105,7 @@ module Integrations def failed_stages_field { title: s_("ChatMessage|Failed stage").pluralize(failed_stages.length), - value: Slack::Messenger::Util::LinkFormatter.format(failed_stages_links), + value: ::Slack::Messenger::Util::LinkFormatter.format(failed_stages_links), short: true } end @@ -113,7 +113,7 @@ module Integrations def failed_jobs_field { title: s_("ChatMessage|Failed job").pluralize(failed_jobs.length), - value: Slack::Messenger::Util::LinkFormatter.format(failed_jobs_links), + value: ::Slack::Messenger::Util::LinkFormatter.format(failed_jobs_links), short: true } end @@ -130,12 +130,12 @@ module Integrations fields = [ { title: ref_type == "tag" ? s_("ChatMessage|Tag") : s_("ChatMessage|Branch"), - value: Slack::Messenger::Util::LinkFormatter.format(ref_link), + value: ::Slack::Messenger::Util::LinkFormatter.format(ref_link), short: true }, { title: s_("ChatMessage|Commit"), - value: Slack::Messenger::Util::LinkFormatter.format(commit_link), + value: ::Slack::Messenger::Util::LinkFormatter.format(commit_link), short: true } ] diff --git a/app/models/integrations/chat_message/push_message.rb b/app/models/integrations/chat_message/push_message.rb index 0952986e923..fabd214633b 100644 --- a/app/models/integrations/chat_message/push_message.rb +++ b/app/models/integrations/chat_message/push_message.rb @@ -49,7 +49,7 @@ module Integrations end def format(string) - Slack::Messenger::Util::LinkFormatter.format(string) + ::Slack::Messenger::Util::LinkFormatter.format(string) end def commit_messages diff --git a/app/models/integrations/chat_message/wiki_page_message.rb b/app/models/integrations/chat_message/wiki_page_message.rb index 9b5275b8c03..00f0f911b0e 100644 --- a/app/models/integrations/chat_message/wiki_page_message.rb +++ b/app/models/integrations/chat_message/wiki_page_message.rb @@ -7,6 +7,7 @@ module Integrations attr_reader :wiki_page_url attr_reader :action attr_reader :description + attr_reader :diff_url def initialize(params) super @@ -16,6 +17,7 @@ module Integrations @title = obj_attr[:title] @wiki_page_url = obj_attr[:url] @description = obj_attr[:message] + @diff_url = obj_attr[:diff_url] @action = case obj_attr[:action] @@ -44,19 +46,23 @@ module Integrations private def message - "#{user_combined_name} #{action} #{wiki_page_link} in #{project_link}: *#{title}*" + "#{user_combined_name} #{action} #{wiki_page_link} (#{diff_link}) in #{project_link}: *#{title}*" end def description_message [{ text: format(@description), color: attachment_color }] end + def diff_link + link('Compare changes', diff_url) + end + def project_link - "[#{project_name}](#{project_url})" + link(project_name, project_url) end def wiki_page_link - "[wiki page](#{wiki_page_url})" + link('wiki page', wiki_page_url) end end end diff --git a/app/models/integrations/custom_issue_tracker.rb b/app/models/integrations/custom_issue_tracker.rb new file mode 100644 index 00000000000..635a9d093e9 --- /dev/null +++ b/app/models/integrations/custom_issue_tracker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Integrations + class CustomIssueTracker < BaseIssueTracker + include ActionView::Helpers::UrlHelper + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? + + def title + s_('IssueTracker|Custom issue tracker') + end + + def description + s_("IssueTracker|Use a custom issue tracker as this project's issue tracker.") + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/custom_issue_tracker'), target: '_blank', rel: 'noopener noreferrer' + s_('IssueTracker|Use a custom issue tracker that is not in the integration list. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'custom_issue_tracker' + end + end +end diff --git a/app/models/integrations/discord.rb b/app/models/integrations/discord.rb new file mode 100644 index 00000000000..ef6d46fd3d3 --- /dev/null +++ b/app/models/integrations/discord.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require "discordrb/webhooks" + +module Integrations + class Discord < BaseChatNotification + include ActionView::Helpers::UrlHelper + + ATTACHMENT_REGEX = /: (?<entry>.*?)\n - (?<name>.*)\n*/.freeze + + def title + s_("DiscordService|Discord Notifications") + end + + def description + s_("DiscordService|Send notifications about project events to a Discord channel.") + end + + def self.to_param + "discord" + end + + def help + docs_link = link_to _('How do I set up this service?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/discord_notifications'), target: '_blank', rel: 'noopener noreferrer' + s_('Send notifications about project events to a Discord channel. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def event_field(event) + # No-op. + end + + def default_channel_placeholder + # No-op. + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push pipeline wiki_page] + end + + def default_fields + [ + { type: "text", name: "webhook", placeholder: "https://discordapp.com/api/webhooks/…", help: "URL to the webhook for the Discord channel." }, + { type: "checkbox", name: "notify_only_broken_pipelines" }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + client = Discordrb::Webhooks::Client.new(url: webhook) + + client.execute do |builder| + builder.add_embed do |embed| + embed.author = Discordrb::Webhooks::EmbedAuthor.new(name: message.user_name, icon_url: message.user_avatar) + embed.description = (message.pretext + "\n" + Array.wrap(message.attachments).join("\n")).gsub(ATTACHMENT_REGEX, " \\k<entry> - \\k<name>\n") + end + end + rescue RestClient::Exception => error + log_error(error.message) + false + end + + def custom_data(data) + super(data).merge(markdown: true) + end + end +end diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb new file mode 100644 index 00000000000..096f7093b8c --- /dev/null +++ b/app/models/integrations/drone_ci.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Integrations + class DroneCi < BaseCi + include ReactiveService + include ServicePushDataValidations + + prop_accessor :drone_url, :token + boolean_accessor :enable_ssl_verification + + validates :drone_url, presence: true, public_url: true, if: :activated? + validates :token, presence: true, if: :activated? + + after_save :compose_service_hook, if: :activated? + + def compose_service_hook + hook = service_hook || build_service_hook + # If using a service template, project may not be available + hook.url = [drone_url, "/api/hook", "?owner=#{project.namespace.full_path}", "&name=#{project.path}", "&access_token=#{token}"].join if project + hook.enable_ssl_verification = !!enable_ssl_verification + hook.save + end + + def execute(data) + case data[:object_kind] + when 'push' + service_hook.execute(data) if push_valid?(data) + when 'merge_request' + service_hook.execute(data) if merge_request_valid?(data) + when 'tag_push' + service_hook.execute(data) if tag_push_valid?(data) + end + end + + def allow_target_ci? + true + end + + def self.supported_events + %w(push merge_request tag_push) + end + + def commit_status_path(sha, ref) + Gitlab::Utils.append_path( + drone_url, + "gitlab/#{project.full_path}/commits/#{sha}?branch=#{Addressable::URI.encode_component(ref.to_s)}&access_token=#{token}") + end + + def commit_status(sha, ref) + with_reactive_cache(sha, ref) { |cached| cached[:commit_status] } + end + + def calculate_reactive_cache(sha, ref) + response = Gitlab::HTTP.try_get(commit_status_path(sha, ref), + verify: enable_ssl_verification, + extra_log_info: { project_id: project_id }) + + status = + if response && response.code == 200 && response['status'] + case response['status'] + when 'killed' + :canceled + when 'failure', 'error' + # Because drone return error if some test env failed + :failed + else + response["status"] + end + else + :error + end + + { commit_status: status } + end + + def build_page(sha, ref) + Gitlab::Utils.append_path( + drone_url, + "gitlab/#{project.full_path}/redirect/commits/#{sha}?branch=#{Addressable::URI.encode_component(ref.to_s)}") + end + + def title + 'Drone' + end + + def description + s_('ProjectService|Run CI/CD pipelines with Drone.') + end + + def self.to_param + 'drone_ci' + end + + def help + s_('ProjectService|Run CI/CD pipelines with Drone.') + end + + def fields + [ + { type: 'text', name: 'token', help: s_('ProjectService|Token for the Drone project.'), required: true }, + { type: 'text', name: 'drone_url', title: s_('ProjectService|Drone server URL'), placeholder: 'http://drone.example.com', required: true }, + { type: 'checkbox', name: 'enable_ssl_verification', title: "Enable SSL verification" } + ] + end + end +end diff --git a/app/models/integrations/ewm.rb b/app/models/integrations/ewm.rb new file mode 100644 index 00000000000..0a4e8d92ed7 --- /dev/null +++ b/app/models/integrations/ewm.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Integrations + class Ewm < BaseIssueTracker + include ActionView::Helpers::UrlHelper + + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? + + def self.reference_pattern(only_long: true) + @reference_pattern ||= %r{(?<issue>\b(bug|task|work item|workitem|rtcwi|defect)\b\s+\d+)}i + end + + def title + 'EWM' + end + + def description + s_("IssueTracker|Use IBM Engineering Workflow Management as this project's issue tracker.") + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/ewm'), target: '_blank', rel: 'noopener noreferrer' + s_("IssueTracker|Use IBM Engineering Workflow Management as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'ewm' + end + + def can_test? + false + end + + def issue_url(iid) + issues_url.gsub(':id', iid.to_s.split(' ')[-1]) + end + end +end diff --git a/app/models/integrations/external_wiki.rb b/app/models/integrations/external_wiki.rb new file mode 100644 index 00000000000..fec435443fa --- /dev/null +++ b/app/models/integrations/external_wiki.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Integrations + class ExternalWiki < Integration + include ActionView::Helpers::UrlHelper + + prop_accessor :external_wiki_url + validates :external_wiki_url, presence: true, public_url: true, if: :activated? + + def title + s_('ExternalWikiService|External wiki') + end + + def description + s_('ExternalWikiService|Link to an external wiki from the sidebar.') + end + + def self.to_param + 'external_wiki' + end + + def fields + [ + { + type: 'text', + name: 'external_wiki_url', + title: s_('ExternalWikiService|External wiki URL'), + placeholder: s_('ExternalWikiService|https://example.com/xxx/wiki/...'), + help: 'Enter the URL to the external wiki.', + required: true + } + ] + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/wiki/index', anchor: 'link-an-external-wiki'), target: '_blank', rel: 'noopener noreferrer' + + s_('Link an external wiki from the project\'s sidebar. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def execute(_data) + response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true) + response.body if response.code == 200 + rescue StandardError + nil + end + + def self.supported_events + %w() + end + end +end diff --git a/app/models/integrations/flowdock.rb b/app/models/integrations/flowdock.rb new file mode 100644 index 00000000000..443f61e65dd --- /dev/null +++ b/app/models/integrations/flowdock.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Integrations + class Flowdock < Integration + include ActionView::Helpers::UrlHelper + + prop_accessor :token + validates :token, presence: true, if: :activated? + + def title + 'Flowdock' + end + + def description + s_('FlowdockService|Send event notifications from GitLab to Flowdock flows.') + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'flowdock'), target: '_blank', rel: 'noopener noreferrer' + s_('FlowdockService|Send event notifications from GitLab to Flowdock flows. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'flowdock' + end + + def fields + [ + { type: 'text', name: 'token', placeholder: s_('FlowdockService|1b609b52537...'), required: true, help: 'Enter your Flowdock token.' } + ] + end + + def self.supported_events + %w(push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + ::Flowdock::Git.post( + data[:ref], + data[:before], + data[:after], + token: token, + repo: project.repository, + repo_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}", + commit_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/-/commit/%s", + diff_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/compare/%s...%s" + ) + end + end +end diff --git a/app/models/integrations/hangouts_chat.rb b/app/models/integrations/hangouts_chat.rb new file mode 100644 index 00000000000..d02cfe4ec56 --- /dev/null +++ b/app/models/integrations/hangouts_chat.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Integrations + class HangoutsChat < BaseChatNotification + include ActionView::Helpers::UrlHelper + + def title + 'Google Chat' + end + + def description + 'Send notifications from GitLab to a room in Google Chat.' + end + + def self.to_param + 'hangouts_chat' + end + + def help + docs_link = link_to _('How do I set up a Google Chat webhook?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/hangouts_chat'), target: '_blank', rel: 'noopener noreferrer' + s_('Before enabling this integration, create a webhook for the room in Google Chat where you want to receive notifications from this project. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def webhook_placeholder + 'https://chat.googleapis.com/v1/spaces…' + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + simple_text = parse_simple_text_message(message) + ::HangoutsChat::Sender.new(webhook).simple(simple_text) + end + + def parse_simple_text_message(message) + header = message.pretext + return header if message.attachments.empty? + + attachment = message.attachments.first + title = format_attachment_title(attachment) + body = attachment[:text] + + [header, title, body].compact.join("\n") + end + + def format_attachment_title(attachment) + return attachment[:title] unless attachment[:title_link] + + "<#{attachment[:title_link]}|#{attachment[:title]}>" + end + end +end diff --git a/app/models/integrations/irker.rb b/app/models/integrations/irker.rb new file mode 100644 index 00000000000..7048dd641ea --- /dev/null +++ b/app/models/integrations/irker.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require 'uri' + +module Integrations + class Irker < Integration + prop_accessor :server_host, :server_port, :default_irc_uri + prop_accessor :recipients, :channels + boolean_accessor :colorize_messages + validates :recipients, presence: true, if: :validate_recipients? + + before_validation :get_channels + + def title + 'Irker (IRC gateway)' + end + + def description + 'Send IRC messages.' + end + + def self.to_param + 'irker' + end + + def self.supported_events + %w(push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + IrkerWorker.perform_async(project_id, channels, + colorize_messages, data, settings) + end + + def settings + { + server_host: server_host.presence || 'localhost', + server_port: server_port.presence || 6659 + } + end + + def fields + [ + { type: 'text', name: 'server_host', placeholder: 'localhost', + help: 'Irker daemon hostname (defaults to localhost)' }, + { type: 'text', name: 'server_port', placeholder: 6659, + help: 'Irker daemon port (defaults to 6659)' }, + { type: 'text', name: 'default_irc_uri', title: 'Default IRC URI', + help: 'A default IRC URI to prepend before each recipient (optional)', + placeholder: 'irc://irc.network.net:6697/' }, + { type: 'textarea', name: 'recipients', + placeholder: 'Recipients/channels separated by whitespaces', required: true, + help: 'Recipients have to be specified with a full URI: '\ + 'irc[s]://irc.network.net[:port]/#channel. Special cases: if '\ + 'you want the channel to be a nickname instead, append ",isnick" to ' \ + 'the channel name; if the channel is protected by a secret password, ' \ + ' append "?key=secretpassword" to the URI (Note that due to a bug, if you ' \ + ' want to use a password, you have to omit the "#" on the channel). If you ' \ + ' specify a default IRC URI to prepend before each recipient, you can just ' \ + ' give a channel name.' }, + { type: 'checkbox', name: 'colorize_messages' } + ] + end + + def help + ' NOTE: Irker does NOT have built-in authentication, which makes it' \ + ' vulnerable to spamming IRC channels if it is hosted outside of a ' \ + ' firewall. Please make sure you run the daemon within a secured network ' \ + ' to prevent abuse. For more details, read: http://www.catb.org/~esr/irker/security.html.' + end + + private + + def get_channels + return true unless activated? + return true if recipients.nil? || recipients.empty? + + map_recipients + + errors.add(:recipients, 'are all invalid') if channels.empty? + true + end + + def map_recipients + self.channels = recipients.split(/\s+/).map do |recipient| + format_channel(recipient) + end + channels.reject!(&:nil?) + end + + def format_channel(recipient) + uri = nil + + # Try to parse the chan as a full URI + begin + uri = consider_uri(URI.parse(recipient)) + rescue URI::InvalidURIError + end + + unless uri.present? && default_irc_uri.nil? + begin + new_recipient = URI.join(default_irc_uri, '/', recipient).to_s + uri = consider_uri(URI.parse(new_recipient)) + rescue StandardError + log_error("Unable to create a valid URL", default_irc_uri: default_irc_uri, recipient: recipient) + end + end + + uri + end + + def consider_uri(uri) + return if uri.scheme.nil? + + # Authorize both irc://domain.com/#chan and irc://domain.com/chan + if uri.is_a?(URI) && uri.scheme[/^ircs?\z/] && !uri.path.nil? + uri.to_s + end + end + end +end diff --git a/app/models/integrations/issue_tracker_data.rb b/app/models/integrations/issue_tracker_data.rb new file mode 100644 index 00000000000..8749075149f --- /dev/null +++ b/app/models/integrations/issue_tracker_data.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Integrations + class IssueTrackerData < ApplicationRecord + include BaseDataFields + + attr_encrypted :project_url, encryption_options + attr_encrypted :issues_url, encryption_options + attr_encrypted :new_issue_url, encryption_options + end +end diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb new file mode 100644 index 00000000000..815e86bcaa1 --- /dev/null +++ b/app/models/integrations/jenkins.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Integrations + class Jenkins < BaseCi + include ActionView::Helpers::UrlHelper + + prop_accessor :jenkins_url, :project_name, :username, :password + + before_update :reset_password + + validates :jenkins_url, presence: true, addressable_url: true, if: :activated? + validates :project_name, presence: true, if: :activated? + validates :username, presence: true, if: ->(service) { service.activated? && service.password_touched? && service.password.present? } + + default_value_for :push_events, true + default_value_for :merge_requests_events, false + default_value_for :tag_push_events, false + + after_save :compose_service_hook, if: :activated? + + def reset_password + # don't reset the password if a new one is provided + if (jenkins_url_changed? || username.blank?) && !password_touched? + self.password = nil + end + end + + def compose_service_hook + hook = service_hook || build_service_hook + hook.url = hook_url + hook.save + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + service_hook.execute(data, "#{data[:object_kind]}_hook") + end + + def test(data) + begin + result = execute(data) + return { success: false, result: result[:message] } if result[:http_status] != 200 + rescue StandardError => error + return { success: false, result: error } + end + + { success: true, result: result[:message] } + end + + def hook_url + url = URI.parse(jenkins_url) + url.path = File.join(url.path || '/', "project/#{project_name}") + url.user = ERB::Util.url_encode(username) unless username.blank? + url.password = ERB::Util.url_encode(password) unless password.blank? + url.to_s + end + + def self.supported_events + %w(push merge_request tag_push) + end + + def title + 'Jenkins' + end + + def description + s_('Run CI/CD pipelines with Jenkins.') + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('integration/jenkins'), target: '_blank', rel: 'noopener noreferrer' + s_('Run CI/CD pipelines with Jenkins when you push to a repository, or when a merge request is created, updated, or merged. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'jenkins' + end + + def fields + [ + { + type: 'text', + name: 'jenkins_url', + title: s_('ProjectService|Jenkins server URL'), + required: true, + placeholder: 'http://jenkins.example.com', + help: s_('The URL of the Jenkins server.') + }, + { + type: 'text', + name: 'project_name', + required: true, + placeholder: 'my_project_name', + help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') + }, + { + type: 'text', + name: 'username', + required: true, + help: s_('The username for the Jenkins server.') + }, + { + type: 'password', + name: 'password', + help: s_('The password for the Jenkins server.'), + non_empty_password_title: s_('ProjectService|Enter new password.'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') + } + ] + end + end +end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb new file mode 100644 index 00000000000..aa143cc28e1 --- /dev/null +++ b/app/models/integrations/jira.rb @@ -0,0 +1,588 @@ +# frozen_string_literal: true + +# Accessible as Project#external_issue_tracker +module Integrations + class Jira < BaseIssueTracker + extend ::Gitlab::Utils::Override + include Gitlab::Routing + include ApplicationHelper + include ActionView::Helpers::AssetUrlHelper + include Gitlab::Utils::StrongMemoize + + PROJECTS_PER_PAGE = 50 + JIRA_CLOUD_HOST = '.atlassian.net' + + ATLASSIAN_REFERRER_GITLAB_COM = { atlOrigin: 'eyJpIjoiY2QyZTJiZDRkNGZhNGZlMWI3NzRkNTBmZmVlNzNiZTkiLCJwIjoianN3LWdpdGxhYi1pbnQifQ' }.freeze + ATLASSIAN_REFERRER_SELF_MANAGED = { atlOrigin: 'eyJpIjoiYjM0MTA4MzUyYTYxNDVkY2IwMzVjOGQ3ZWQ3NzMwM2QiLCJwIjoianN3LWdpdGxhYlNNLWludCJ9' }.freeze + + validates :url, public_url: true, presence: true, if: :activated? + validates :api_url, public_url: true, allow_blank: true + validates :username, presence: true, if: :activated? + validates :password, presence: true, if: :activated? + + validates :jira_issue_transition_id, + format: { with: Gitlab::Regex.jira_transition_id_regex, message: s_("JiraService|IDs must be a list of numbers that can be split with , or ;") }, + allow_blank: true + + # Jira Cloud version is deprecating authentication via username and password. + # We should use username/password for Jira Server and email/api_token for Jira Cloud, + # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. + + # TODO: we can probably just delegate as part of + # https://gitlab.com/gitlab-org/gitlab/issues/29404 + data_field :username, :password, :url, :api_url, :jira_issue_transition_automatic, :jira_issue_transition_id, :project_key, :issues_enabled, + :vulnerabilities_enabled, :vulnerabilities_issuetype + + before_update :reset_password + after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? + + enum comment_detail: { + standard: 1, + all_details: 2 + } + + # When these are false GitLab does not create cross reference + # comments on Jira except when an issue gets transitioned. + def self.supported_events + %w(commit merge_request) + end + + def self.supported_event_actions + %w(comment) + end + + # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 + def self.reference_pattern(only_long: true) + @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ + end + + def initialize_properties + {} + end + + def data_fields + jira_tracker_data || self.build_jira_tracker_data + end + + def reset_password + data_fields.password = nil if reset_password? + end + + def set_default_data + return unless issues_tracker.present? + + return if url + + data_fields.url ||= issues_tracker['url'] + data_fields.api_url ||= issues_tracker['api_url'] + end + + def options + url = URI.parse(client_url) + + { + username: username&.strip, + password: password, + site: URI.join(url, '/').to_s.delete_suffix('/'), # Intended to find the root + context_path: (url.path.presence || '/').delete_suffix('/'), + auth_type: :basic, + read_timeout: 120, + use_cookies: true, + additional_cookies: ['OBBasicAuth=fromDialog'], + use_ssl: url.scheme == 'https' + } + end + + def client + @client ||= begin + JIRA::Client.new(options).tap do |client| + # Replaces JIRA default http client with our implementation + client.request_client = Gitlab::Jira::HttpClient.new(client.options) + end + end + end + + def help + jira_doc_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_url('integration/jira/index.html') } + s_("JiraService|You need to configure Jira before enabling this integration. For more details, read the %{jira_doc_link_start}Jira integration documentation%{link_end}.") % { jira_doc_link_start: jira_doc_link_start, link_end: '</a>'.html_safe } + end + + def title + 'Jira' + end + + def description + s_("JiraService|Use Jira as this project's issue tracker.") + end + + def self.to_param + 'jira' + end + + def fields + [ + { + type: 'text', + name: 'url', + title: s_('JiraService|Web URL'), + placeholder: 'https://jira.example.com', + help: s_('JiraService|Base URL of the Jira instance.'), + required: true + }, + { + type: 'text', + name: 'api_url', + title: s_('JiraService|Jira API URL'), + help: s_('JiraService|If different from Web URL.') + }, + { + type: 'text', + name: 'username', + title: s_('JiraService|Username or Email'), + help: s_('JiraService|Use a username for server version and an email for cloud version.'), + required: true + }, + { + type: 'password', + name: 'password', + title: s_('JiraService|Password or API token'), + non_empty_password_title: s_('JiraService|Enter new password or API token'), + non_empty_password_help: s_('JiraService|Leave blank to use your current password or API token.'), + help: s_('JiraService|Use a password for server version and an API token for cloud version.'), + required: true + } + ] + end + + def web_url(path = nil, **params) + return unless url.present? + + if Gitlab.com? + params.merge!(ATLASSIAN_REFERRER_GITLAB_COM) unless Gitlab.staging? + else + params.merge!(ATLASSIAN_REFERRER_SELF_MANAGED) unless Gitlab.dev_or_test_env? + end + + url = Addressable::URI.parse(self.url) + url.path = url.path.delete_suffix('/') + url.path << "/#{path.delete_prefix('/').delete_suffix('/')}" if path.present? + url.query_values = (url.query_values || {}).merge(params) + url.query_values = nil if url.query_values.empty? + + url.to_s + end + + override :project_url + def project_url + web_url + end + + override :issues_url + def issues_url + web_url('browse/:id') + end + + override :new_issue_url + def new_issue_url + web_url('secure/CreateIssue!default.jspa') + end + + alias_method :original_url, :url + def url + original_url&.delete_suffix('/') + end + + alias_method :original_api_url, :api_url + def api_url + original_api_url&.delete_suffix('/') + end + + def execute(push) + # This method is a no-op, because currently Integrations::Jira does not + # support any events. + end + + def find_issue(issue_key, rendered_fields: false, transitions: false) + expands = [] + expands << 'renderedFields' if rendered_fields + expands << 'transitions' if transitions + options = { expand: expands.join(',') } if expands.any? + + jira_request { client.Issue.find(issue_key, options || {}) } + end + + def close_issue(entity, external_issue, current_user) + issue = find_issue(external_issue.iid, transitions: jira_issue_transition_automatic) + + return if issue.nil? || has_resolution?(issue) || !issue_transition_enabled? + + commit_id = case entity + when Commit then entity.id + when MergeRequest then entity.diff_head_sha + end + + commit_url = build_entity_url(:commit, commit_id) + + # Depending on the Jira project's workflow, a comment during transition + # may or may not be allowed. Refresh the issue after transition and check + # if it is closed, so we don't have one comment for every commit. + issue = find_issue(issue.key) if transition_issue(issue) + add_issue_solved_comment(issue, commit_id, commit_url) if has_resolution?(issue) + log_usage(:close_issue, current_user) + end + + override :create_cross_reference_note + def create_cross_reference_note(mentioned, noteable, author) + unless can_cross_reference?(noteable) + return s_("JiraService|Events for %{noteable_model_name} are disabled.") % { noteable_model_name: noteable.model_name.plural.humanize(capitalize: false) } + end + + jira_issue = find_issue(mentioned.id) + + return unless jira_issue.present? + + noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id + noteable_type = noteable_name(noteable) + entity_url = build_entity_url(noteable_type, noteable_id) + entity_meta = build_entity_meta(noteable) + + data = { + user: { + name: author.name, + url: resource_url(user_path(author)) + }, + project: { + name: project.full_path, + url: resource_url(project_path(project)) + }, + entity: { + id: entity_meta[:id], + name: noteable_type.humanize.downcase, + url: entity_url, + title: noteable.title, + description: entity_meta[:description], + branch: entity_meta[:branch] + } + } + + add_comment(data, jira_issue).tap { log_usage(:cross_reference, author) } + end + + def valid_connection? + test(nil)[:success] + end + + def test(_) + result = server_info + success = result.present? + result = @error&.message unless success + + { success: success, result: result } + end + + override :support_close_issue? + def support_close_issue? + true + end + + override :support_cross_reference? + def support_cross_reference? + true + end + + def issue_transition_enabled? + jira_issue_transition_automatic || jira_issue_transition_id.present? + end + + private + + def server_info + strong_memoize(:server_info) do + client_url.present? ? jira_request { client.ServerInfo.all.attrs } : nil + end + end + + def can_cross_reference?(noteable) + case noteable + when Commit then commit_events + when MergeRequest then merge_requests_events + else true + end + end + + # jira_issue_transition_id can have multiple values split by , or ; + # the issue is transitioned at the order given by the user + # if any transition fails it will log the error message and stop the transition sequence + def transition_issue(issue) + return transition_issue_to_done(issue) if jira_issue_transition_automatic + + jira_issue_transition_id.scan(Gitlab::Regex.jira_transition_id_regex).all? do |transition_id| + transition_issue_to_id(issue, transition_id) + end + end + + def transition_issue_to_id(issue, transition_id) + issue.transitions.build.save!( + transition: { id: transition_id } + ) + + true + rescue StandardError => error + log_error( + "Issue transition failed", + error: { + exception_class: error.class.name, + exception_message: error.message, + exception_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(error.backtrace) + }, + client_url: client_url + ) + + false + end + + def transition_issue_to_done(issue) + transitions = issue.transitions rescue [] + + transition = transitions.find do |transition| + status = transition&.to&.statusCategory + status && status['key'] == 'done' + end + + return false unless transition + + transition_issue_to_id(issue, transition.id) + end + + def log_usage(action, user) + key = "i_ecosystem_jira_service_#{action}" + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user.id) + end + + def add_issue_solved_comment(issue, commit_id, commit_url) + link_title = "Solved by commit #{commit_id}." + comment = "Issue solved with [#{commit_id}|#{commit_url}]." + link_props = build_remote_link_props(url: commit_url, title: link_title, resolved: true) + send_message(issue, comment, link_props) + end + + def add_comment(data, issue) + entity_name = data[:entity][:name] + entity_url = data[:entity][:url] + entity_title = data[:entity][:title] + + message = comment_message(data) + link_title = "#{entity_name.capitalize} - #{entity_title}" + link_props = build_remote_link_props(url: entity_url, title: link_title) + + unless comment_exists?(issue, message) + send_message(issue, message, link_props) + end + end + + def comment_message(data) + user_link = build_jira_link(data[:user][:name], data[:user][:url]) + + entity = data[:entity] + entity_ref = all_details? ? "#{entity[:name]} #{entity[:id]}" : "a #{entity[:name]}" + entity_link = build_jira_link(entity_ref, entity[:url]) + + project_link = build_jira_link(project.full_name, Gitlab::Routing.url_helpers.project_url(project)) + branch = + if entity[:branch].present? + s_('JiraService| on branch %{branch_link}') % { + branch_link: build_jira_link(entity[:branch], project_tree_url(project, entity[:branch])) + } + end + + entity_message = entity[:description].presence if all_details? + entity_message ||= entity[:title].chomp + + s_('JiraService|%{user_link} mentioned this issue in %{entity_link} of %{project_link}%{branch}:{quote}%{entity_message}{quote}') % { + user_link: user_link, + entity_link: entity_link, + project_link: project_link, + branch: branch, + entity_message: entity_message + } + end + + def build_jira_link(title, url) + "[#{title}|#{url}]" + end + + def has_resolution?(issue) + issue.respond_to?(:resolution) && issue.resolution.present? + end + + def comment_exists?(issue, message) + comments = jira_request { issue.comments } + + comments.present? && comments.any? { |comment| comment.body.include?(message) } + end + + def send_message(issue, message, remote_link_props) + return unless client_url.present? + + jira_request do + remote_link = find_remote_link(issue, remote_link_props[:object][:url]) + + create_issue_comment(issue, message) unless remote_link + remote_link ||= issue.remotelink.build + remote_link.save!(remote_link_props) + + log_info("Successfully posted", client_url: client_url) + "SUCCESS: Successfully posted to #{client_url}." + end + end + + def create_issue_comment(issue, message) + return unless comment_on_event_enabled + + issue.comments.build.save!(body: message) + end + + def find_remote_link(issue, url) + links = jira_request { issue.remotelink.all } + return unless links + + links.find { |link| link.object["url"] == url } + end + + def build_remote_link_props(url:, title:, resolved: false) + status = { + resolved: resolved + } + + { + GlobalID: 'GitLab', + relationship: 'mentioned on', + object: { + url: url, + title: title, + status: status, + icon: { + title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.base_url) + } + } + } + end + + def resource_url(resource) + "#{Settings.gitlab.base_url.chomp("/")}#{resource}" + end + + def build_entity_url(noteable_type, entity_id) + polymorphic_url( + [ + self.project, + noteable_type.to_sym + ], + id: entity_id, + host: Settings.gitlab.base_url + ) + end + + def build_entity_meta(noteable) + if noteable.is_a?(Commit) + { + id: noteable.short_id, + description: noteable.safe_message, + branch: noteable.ref_names(project.repository).first + } + elsif noteable.is_a?(MergeRequest) + { + id: noteable.to_reference, + branch: noteable.source_branch + } + else + {} + end + end + + def noteable_name(noteable) + name = noteable.model_name.singular + + # ProjectSnippet inherits from Snippet class so it causes + # routing error building the URL. + name == "project_snippet" ? "snippet" : name + end + + # Handle errors when doing Jira API calls + def jira_request + yield + rescue StandardError => error + @error = error + log_error("Error sending message", client_url: client_url, error: @error.message) + nil + end + + def client_url + api_url.presence || url + end + + def reset_password? + # don't reset the password if a new one is provided + return false if password_touched? + return true if api_url_changed? + return false if api_url.present? + + url_changed? + end + + def update_deployment_type? + (api_url_changed? || url_changed? || username_changed? || password_changed?) && + can_test? + end + + def update_deployment_type + clear_memoization(:server_info) # ensure we run the request when we try to update deployment type + results = server_info + + unless results.present? + Gitlab::AppLogger.warn(message: "Jira API returned no ServerInfo, setting deployment_type from URL", server_info: results, url: client_url) + + return set_deployment_type_from_url + end + + if jira_cloud? + data_fields.deployment_cloud! + else + data_fields.deployment_server! + end + end + + def jira_cloud? + server_info['deploymentType'] == 'Cloud' || URI(client_url).hostname.end_with?(JIRA_CLOUD_HOST) + end + + def set_deployment_type_from_url + # This shouldn't happen but of course it will happen when an integration is removed. + # Instead of deleting the integration we set all fields to null + # and mark it as inactive + return data_fields.deployment_unknown! unless client_url + + # If API-based detection methods fail here then + # we can only assume it's either Cloud or Server + # based on the URL being *.atlassian.net + + if URI(client_url).hostname.end_with?(JIRA_CLOUD_HOST) + data_fields.deployment_cloud! + else + data_fields.deployment_server! + end + end + + def self.event_description(event) + case event + when "merge_request", "merge_request_events" + s_("JiraService|Jira comments are created when an issue is referenced in a merge request.") + when "commit", "commit_events" + s_("JiraService|Jira comments are created when an issue is referenced in a commit.") + end + end + end +end + +Integrations::Jira.prepend_mod_with('Integrations::Jira') diff --git a/app/models/integrations/jira_tracker_data.rb b/app/models/integrations/jira_tracker_data.rb new file mode 100644 index 00000000000..74352393b43 --- /dev/null +++ b/app/models/integrations/jira_tracker_data.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Integrations + class JiraTrackerData < ApplicationRecord + include BaseDataFields + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + attr_encrypted :username, encryption_options + attr_encrypted :password, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end +end diff --git a/app/models/integrations/mattermost.rb b/app/models/integrations/mattermost.rb new file mode 100644 index 00000000000..07a5086b8e9 --- /dev/null +++ b/app/models/integrations/mattermost.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Integrations + class Mattermost < BaseChatNotification + include SlackMattermostNotifier + include ActionView::Helpers::UrlHelper + + def title + s_('Mattermost notifications') + end + + def description + s_('Send notifications about project events to Mattermost channels.') + end + + def self.to_param + 'mattermost' + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/mattermost'), target: '_blank', rel: 'noopener noreferrer' + s_('Send notifications about project events to Mattermost channels. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def default_channel_placeholder + 'my-channel' + end + + def webhook_placeholder + 'http://mattermost.example.com/hooks/' + end + end +end diff --git a/app/models/integrations/mattermost_slash_commands.rb b/app/models/integrations/mattermost_slash_commands.rb new file mode 100644 index 00000000000..6cd664da9e7 --- /dev/null +++ b/app/models/integrations/mattermost_slash_commands.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Integrations + class MattermostSlashCommands < BaseSlashCommands + include Ci::TriggersHelper + + prop_accessor :token + + def can_test? + false + end + + def title + 'Mattermost slash commands' + end + + def description + "Perform common tasks with slash commands." + end + + def self.to_param + 'mattermost_slash_commands' + end + + def configure(user, params) + token = ::Mattermost::Command.new(user) + .create(command(params)) + + update(active: true, token: token) if token + rescue ::Mattermost::Error => e + [false, e.message] + end + + def list_teams(current_user) + [::Mattermost::Team.new(current_user).all, nil] + rescue ::Mattermost::Error => e + [[], e.message] + end + + def chat_responder + ::Gitlab::Chat::Responder::Mattermost + end + + private + + def command(params) + pretty_project_name = project.full_name + + params.merge( + auto_complete: true, + auto_complete_desc: "Perform common operations on: #{pretty_project_name}", + auto_complete_hint: '[help]', + description: "Perform common operations on: #{pretty_project_name}", + display_name: "GitLab / #{pretty_project_name}", + method: 'P', + username: 'GitLab') + end + end +end diff --git a/app/models/integrations/microsoft_teams.rb b/app/models/integrations/microsoft_teams.rb new file mode 100644 index 00000000000..91e6800f03c --- /dev/null +++ b/app/models/integrations/microsoft_teams.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Integrations + class MicrosoftTeams < BaseChatNotification + def title + 'Microsoft Teams notifications' + end + + def description + 'Send notifications about project events to Microsoft Teams.' + end + + def self.to_param + 'microsoft_teams' + end + + def help + '<p>Use this service to send notifications about events in GitLab projects to your Microsoft Teams channels. <a href="https://docs.gitlab.com/ee/user/project/integrations/microsoft_teams.html">How do I configure this integration?</a></p>' + end + + def webhook_placeholder + 'https://outlook.office.com/webhook/…' + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, + { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'If selected, successful pipelines do not trigger a notification event.' }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + ::MicrosoftTeams::Notifier.new(webhook).ping( + title: message.project_name, + summary: message.summary, + activity: message.activity, + attachments: message.attachments + ) + end + + def custom_data(data) + super(data).merge(markdown: true) + end + end +end diff --git a/app/models/integrations/mock_ci.rb b/app/models/integrations/mock_ci.rb new file mode 100644 index 00000000000..d31f6381767 --- /dev/null +++ b/app/models/integrations/mock_ci.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +# For an example companion mocking service, see https://gitlab.com/gitlab-org/gitlab-mock-ci-service +module Integrations + class MockCi < BaseCi + ALLOWED_STATES = %w[failed canceled running pending success success-with-warnings skipped not_found].freeze + + prop_accessor :mock_service_url + validates :mock_service_url, presence: true, public_url: true, if: :activated? + + def title + 'MockCI' + end + + def description + 'Mock an external CI' + end + + def self.to_param + 'mock_ci' + end + + def fields + [ + { + type: 'text', + name: 'mock_service_url', + title: s_('ProjectService|Mock service URL'), + placeholder: 'http://localhost:4004', + required: true + } + ] + end + + # Return complete url to build page + # + # Ex. + # http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c + # + def build_page(sha, ref) + Gitlab::Utils.append_path( + mock_service_url, + "#{project.namespace.path}/#{project.path}/status/#{sha}") + end + + # Return string with build status or :error symbol + # + # Allowed states: 'success', 'failed', 'running', 'pending', 'skipped' + # + # Ex. + # @service.commit_status('13be4ac', 'master') + # # => 'success' + # + # @service.commit_status('2abe4ac', 'dev') + # # => 'running' + # + def commit_status(sha, ref) + response = Gitlab::HTTP.get(commit_status_path(sha), verify: false) + read_commit_status(response) + rescue Errno::ECONNREFUSED + :error + end + + def commit_status_path(sha) + Gitlab::Utils.append_path( + mock_service_url, + "#{project.namespace.path}/#{project.path}/status/#{sha}.json") + end + + def read_commit_status(response) + return :error unless response.code == 200 || response.code == 404 + + status = if response.code == 404 + 'pending' + else + response['status'] + end + + if status.present? && ALLOWED_STATES.include?(status) + status + else + :error + end + end + + def can_test? + false + end + end +end diff --git a/app/models/integrations/open_project.rb b/app/models/integrations/open_project.rb new file mode 100644 index 00000000000..e4cfb24151a --- /dev/null +++ b/app/models/integrations/open_project.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Integrations + class OpenProject < BaseIssueTracker + validates :url, public_url: true, presence: true, if: :activated? + validates :api_url, public_url: true, allow_blank: true, if: :activated? + validates :token, presence: true, if: :activated? + validates :project_identifier_code, presence: true, if: :activated? + + data_field :url, :api_url, :token, :closed_status_id, :project_identifier_code + + def data_fields + open_project_tracker_data || self.build_open_project_tracker_data + end + + def self.to_param + 'open_project' + end + end +end diff --git a/app/models/integrations/open_project_tracker_data.rb b/app/models/integrations/open_project_tracker_data.rb new file mode 100644 index 00000000000..b3f2618b94f --- /dev/null +++ b/app/models/integrations/open_project_tracker_data.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Integrations + class OpenProjectTrackerData < ApplicationRecord + include BaseDataFields + + # When the Open Project is fresh installed, the default closed status id is "13" based on current version: v8. + DEFAULT_CLOSED_STATUS_ID = "13" + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + attr_encrypted :token, encryption_options + + def closed_status_id + super || DEFAULT_CLOSED_STATUS_ID + end + end +end diff --git a/app/models/integrations/packagist.rb b/app/models/integrations/packagist.rb new file mode 100644 index 00000000000..b597bd11175 --- /dev/null +++ b/app/models/integrations/packagist.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Integrations + class Packagist < Integration + prop_accessor :username, :token, :server + + validates :username, presence: true, if: :activated? + validates :token, presence: true, if: :activated? + + default_value_for :push_events, true + default_value_for :tag_push_events, true + + after_save :compose_service_hook, if: :activated? + + def title + 'Packagist' + end + + def description + s_('Integrations|Update your Packagist projects.') + end + + def self.to_param + 'packagist' + end + + def fields + [ + { type: 'text', name: 'username', placeholder: '', required: true }, + { type: 'text', name: 'token', placeholder: '', required: true }, + { type: 'text', name: 'server', placeholder: 'https://packagist.org', required: false } + ] + end + + def self.supported_events + %w(push merge_request tag_push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + service_hook.execute(data) + end + + def test(data) + begin + result = execute(data) + return { success: false, result: result[:message] } if result[:http_status] != 202 + rescue StandardError => error + return { success: false, result: error } + end + + { success: true, result: result[:message] } + end + + def compose_service_hook + hook = service_hook || build_service_hook + hook.url = hook_url + hook.save + end + + def hook_url + base_url = server.presence || 'https://packagist.org' + "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}" + end + end +end diff --git a/app/models/integrations/pipelines_email.rb b/app/models/integrations/pipelines_email.rb new file mode 100644 index 00000000000..585bc14242a --- /dev/null +++ b/app/models/integrations/pipelines_email.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Integrations + class PipelinesEmail < Integration + include NotificationBranchSelection + + prop_accessor :recipients, :branches_to_be_notified + boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch + validates :recipients, presence: true, if: :validate_recipients? + + def initialize_properties + if properties.nil? + self.properties = {} + self.notify_only_broken_pipelines = true + self.branches_to_be_notified = "default" + elsif !self.notify_only_default_branch.nil? + # In older versions, there was only a boolean property named + # `notify_only_default_branch`. Now we have a string property named + # `branches_to_be_notified`. Instead of doing a background migration, we + # opted to set a value for the new property based on the old one, if + # users hasn't specified one already. When users edit the service and + # selects a value for this new property, it will override everything. + + self.branches_to_be_notified ||= notify_only_default_branch? ? "default" : "all" + end + end + + def title + _('Pipeline status emails') + end + + def description + _('Email the pipeline status to a list of recipients.') + end + + def self.to_param + 'pipelines_email' + end + + def self.supported_events + %w[pipeline] + end + + def self.default_test_event + 'pipeline' + end + + def execute(data, force: false) + return unless supported_events.include?(data[:object_kind]) + return unless force || should_pipeline_be_notified?(data) + + all_recipients = retrieve_recipients(data) + + return unless all_recipients.any? + + pipeline_id = data[:object_attributes][:id] + PipelineNotificationWorker.new.perform(pipeline_id, recipients: all_recipients) + end + + def can_test? + project&.ci_pipelines&.any? + end + + def fields + [ + { type: 'textarea', + name: 'recipients', + help: _('Comma-separated list of email addresses.'), + required: true }, + { type: 'checkbox', + name: 'notify_only_broken_pipelines' }, + { type: 'select', + name: 'branches_to_be_notified', + choices: branch_choices } + ] + end + + def test(data) + result = execute(data, force: true) + + { success: true, result: result } + rescue StandardError => error + { success: false, result: error } + end + + def should_pipeline_be_notified?(data) + notify_for_branch?(data) && notify_for_pipeline?(data) + end + + def notify_for_pipeline?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end + + def retrieve_recipients(data) + recipients.to_s.split(/[,\r\n ]+/).reject(&:empty?) + end + end +end diff --git a/app/models/integrations/pivotaltracker.rb b/app/models/integrations/pivotaltracker.rb new file mode 100644 index 00000000000..46f97cc3c6b --- /dev/null +++ b/app/models/integrations/pivotaltracker.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Integrations + class Pivotaltracker < Integration + API_ENDPOINT = 'https://www.pivotaltracker.com/services/v5/source_commits' + + prop_accessor :token, :restrict_to_branch + validates :token, presence: true, if: :activated? + + def title + 'PivotalTracker' + end + + def description + s_('PivotalTrackerService|Add commit messages as comments to PivotalTracker stories.') + end + + def self.to_param + 'pivotaltracker' + end + + def fields + [ + { + type: 'text', + name: 'token', + placeholder: s_('PivotalTrackerService|Pivotal Tracker API token.'), + required: true + }, + { + type: 'text', + name: 'restrict_to_branch', + placeholder: s_('PivotalTrackerService|Comma-separated list of branches which will be ' \ + 'automatically inspected. Leave blank to include all branches.') + } + ] + end + + def self.supported_events + %w(push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + return unless allowed_branch?(data[:ref]) + + data[:commits].each do |commit| + message = { + 'source_commit' => { + 'commit_id' => commit[:id], + 'author' => commit[:author][:name], + 'url' => commit[:url], + 'message' => commit[:message] + } + } + Gitlab::HTTP.post( + API_ENDPOINT, + body: message.to_json, + headers: { + 'Content-Type' => 'application/json', + 'X-TrackerToken' => token + } + ) + end + end + + private + + def allowed_branch?(ref) + return true unless ref.present? && restrict_to_branch.present? + + branch = Gitlab::Git.ref_name(ref) + allowed_branches = restrict_to_branch.split(',').map(&:strip) + + branch.present? && allowed_branches.include?(branch) + end + end +end diff --git a/app/models/integrations/pushover.rb b/app/models/integrations/pushover.rb new file mode 100644 index 00000000000..b0cadc7ef4e --- /dev/null +++ b/app/models/integrations/pushover.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Integrations + class Pushover < Integration + BASE_URI = 'https://api.pushover.net/1' + + prop_accessor :api_key, :user_key, :device, :priority, :sound + validates :api_key, :user_key, :priority, presence: true, if: :activated? + + def title + 'Pushover' + end + + def description + s_('PushoverService|Get real-time notifications on your device.') + end + + def self.to_param + 'pushover' + end + + def fields + [ + { type: 'text', name: 'api_key', title: _('API key'), placeholder: s_('PushoverService|Your application key'), required: true }, + { type: 'text', name: 'user_key', placeholder: s_('PushoverService|Your user key'), required: true }, + { type: 'text', name: 'device', placeholder: s_('PushoverService|Leave blank for all active devices') }, + { type: 'select', name: 'priority', required: true, choices: + [ + [s_('PushoverService|Lowest Priority'), -2], + [s_('PushoverService|Low Priority'), -1], + [s_('PushoverService|Normal Priority'), 0], + [s_('PushoverService|High Priority'), 1] + ], + default_choice: 0 }, + { type: 'select', name: 'sound', choices: + [ + ['Device default sound', nil], + ['Pushover (default)', 'pushover'], + %w(Bike bike), + %w(Bugle bugle), + ['Cash Register', 'cashregister'], + %w(Classical classical), + %w(Cosmic cosmic), + %w(Falling falling), + %w(Gamelan gamelan), + %w(Incoming incoming), + %w(Intermission intermission), + %w(Magic magic), + %w(Mechanical mechanical), + ['Piano Bar', 'pianobar'], + %w(Siren siren), + ['Space Alarm', 'spacealarm'], + ['Tug Boat', 'tugboat'], + ['Alien Alarm (long)', 'alien'], + ['Climb (long)', 'climb'], + ['Persistent (long)', 'persistent'], + ['Pushover Echo (long)', 'echo'], + ['Up Down (long)', 'updown'], + ['None (silent)', 'none'] + ] } + ] + end + + def self.supported_events + %w(push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + ref = Gitlab::Git.ref_name(data[:ref]) + before = data[:before] + after = data[:after] + + message = + if Gitlab::Git.blank_ref?(before) + s_("PushoverService|%{user_name} pushed new branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } + elsif Gitlab::Git.blank_ref?(after) + s_("PushoverService|%{user_name} deleted branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } + else + s_("PushoverService|%{user_name} push to branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } + end + + if data[:total_commits_count] > 0 + message = [message, s_("PushoverService|Total commits count: %{total_commits_count}") % { total_commits_count: data[:total_commits_count] }].join("\n") + end + + pushover_data = { + token: api_key, + user: user_key, + device: device, + priority: priority, + title: "#{project.full_name}", + message: message, + url: data[:project][:web_url], + url_title: s_("PushoverService|See project %{project_full_name}") % { project_full_name: project.full_name } + } + + # Sound parameter MUST NOT be sent to API if not selected + if sound + pushover_data[:sound] = sound + end + + Gitlab::HTTP.post('/messages.json', base_uri: BASE_URI, body: pushover_data) + end + end +end diff --git a/app/models/integrations/redmine.rb b/app/models/integrations/redmine.rb new file mode 100644 index 00000000000..990b538f294 --- /dev/null +++ b/app/models/integrations/redmine.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Integrations + class Redmine < BaseIssueTracker + include ActionView::Helpers::UrlHelper + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? + + def title + 'Redmine' + end + + def description + s_("IssueTracker|Use Redmine as this project's issue tracker.") + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/redmine'), target: '_blank', rel: 'noopener noreferrer' + s_('IssueTracker|Use Redmine as the issue tracker. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'redmine' + end + end +end diff --git a/app/models/integrations/slack.rb b/app/models/integrations/slack.rb new file mode 100644 index 00000000000..0381db3a67e --- /dev/null +++ b/app/models/integrations/slack.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Integrations + class Slack < BaseChatNotification + include SlackMattermostNotifier + extend ::Gitlab::Utils::Override + + SUPPORTED_EVENTS_FOR_USAGE_LOG = %w[ + push issue confidential_issue merge_request note confidential_note + tag_push wiki_page deployment + ].freeze + + prop_accessor EVENT_CHANNEL['alert'] + + def title + 'Slack notifications' + end + + def description + 'Send notifications about project events to Slack.' + end + + def self.to_param + 'slack' + end + + def default_channel_placeholder + _('#general, #development') + end + + def webhook_placeholder + 'https://hooks.slack.com/services/…' + end + + def supported_events + additional = [] + additional << 'alert' + + super + additional + end + + def get_message(object_kind, data) + return Integrations::ChatMessage::AlertMessage.new(data) if object_kind == 'alert' + + super + end + + override :log_usage + def log_usage(event, user_id) + return unless user_id + + return unless SUPPORTED_EVENTS_FOR_USAGE_LOG.include?(event) + + key = "i_ecosystem_slack_service_#{event}_notification" + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) + end + end +end diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb new file mode 100644 index 00000000000..ff1f806df45 --- /dev/null +++ b/app/models/integrations/slack_slash_commands.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Integrations + class SlackSlashCommands < BaseSlashCommands + include Ci::TriggersHelper + + def title + 'Slack slash commands' + end + + def description + "Perform common operations in Slack" + end + + def self.to_param + 'slack_slash_commands' + end + + def trigger(params) + # Format messages to be Slack-compatible + super.tap do |result| + result[:text] = format(result[:text]) if result.is_a?(Hash) + end + end + + def chat_responder + ::Gitlab::Chat::Responder::Slack + end + + private + + def format(text) + ::Slack::Messenger::Util::LinkFormatter.format(text) if text + end + end +end diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb new file mode 100644 index 00000000000..8284d5963ae --- /dev/null +++ b/app/models/integrations/teamcity.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +module Integrations + class Teamcity < BaseCi + include ReactiveService + include ServicePushDataValidations + + prop_accessor :teamcity_url, :build_type, :username, :password + + validates :teamcity_url, presence: true, public_url: true, if: :activated? + validates :build_type, presence: true, if: :activated? + validates :username, + presence: true, + if: ->(service) { service.activated? && service.password } + validates :password, + presence: true, + if: ->(service) { service.activated? && service.username } + + attr_accessor :response + + after_save :compose_service_hook, if: :activated? + before_update :reset_password + + class << self + def to_param + 'teamcity' + end + + def supported_events + %w(push merge_request) + end + + def event_description(event) + case event + when 'push', 'push_events' + 'TeamCity CI will be triggered after every push to the repository except branch delete' + when 'merge_request', 'merge_request_events' + 'TeamCity CI will be triggered after a merge request has been created or updated' + end + end + end + + def compose_service_hook + hook = service_hook || build_service_hook + hook.save + end + + def reset_password + if teamcity_url_changed? && !password_touched? + self.password = nil + end + end + + def title + 'JetBrains TeamCity' + end + + def description + s_('ProjectService|Run CI/CD pipelines with JetBrains TeamCity.') + end + + def help + s_('To run CI/CD pipelines with JetBrains TeamCity, input the GitLab project details in the TeamCity project Version Control Settings.') + end + + def fields + [ + { + type: 'text', + name: 'teamcity_url', + title: s_('ProjectService|TeamCity server URL'), + placeholder: 'https://teamcity.example.com', + required: true + }, + { + type: 'text', + name: 'build_type', + help: s_('ProjectService|The build configuration ID of the TeamCity project.'), + required: true + }, + { + type: 'text', + name: 'username', + help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') + }, + { + type: 'password', + name: 'password', + non_empty_password_title: s_('ProjectService|Enter new password'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password') + } + ] + end + + def build_page(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:build_page] } + end + + def commit_status(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + end + + def calculate_reactive_cache(sha, ref) + response = get_path("httpAuth/app/rest/builds/branch:unspecified:any,revision:#{sha}") + + if response + { build_page: read_build_page(response), commit_status: read_commit_status(response) } + else + { build_page: teamcity_url, commit_status: :error } + end + end + + def execute(data) + case data[:object_kind] + when 'push' + execute_push(data) + when 'merge_request' + execute_merge_request(data) + end + end + + private + + def execute_push(data) + branch = Gitlab::Git.ref_name(data[:ref]) + post_to_build_queue(data, branch) if push_valid?(data) + end + + def execute_merge_request(data) + branch = data[:object_attributes][:source_branch] + post_to_build_queue(data, branch) if merge_request_valid?(data) + end + + def read_build_page(response) + if response.code != 200 + # If actual build link can't be determined, + # send user to build summary page. + build_url("viewLog.html?buildTypeId=#{build_type}") + else + # If actual build link is available, go to build result page. + built_id = response['build']['id'] + build_url("viewLog.html?buildId=#{built_id}&buildTypeId=#{build_type}") + end + end + + def read_commit_status(response) + return :error unless response.code == 200 || response.code == 404 + + status = if response.code == 404 + 'Pending' + else + response['build']['status'] + end + + return :error unless status.present? + + if status.include?('SUCCESS') + 'success' + elsif status.include?('FAILURE') + 'failed' + elsif status.include?('Pending') + 'pending' + else + :error + end + end + + def build_url(path) + Gitlab::Utils.append_path(teamcity_url, path) + end + + def get_path(path) + Gitlab::HTTP.try_get(build_url(path), verify: false, basic_auth: basic_auth, extra_log_info: { project_id: project_id }) + end + + def post_to_build_queue(data, branch) + Gitlab::HTTP.post( + build_url('httpAuth/app/rest/buildQueue'), + body: "<build branchName=#{branch.encode(xml: :attr)}>"\ + "<buildType id=#{build_type.encode(xml: :attr)}/>"\ + '</build>', + headers: { 'Content-type' => 'application/xml' }, + basic_auth: basic_auth + ) + end + + def basic_auth + { username: username, password: password } + end + end +end diff --git a/app/models/integrations/unify_circuit.rb b/app/models/integrations/unify_circuit.rb new file mode 100644 index 00000000000..03363c7c8b0 --- /dev/null +++ b/app/models/integrations/unify_circuit.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Integrations + class UnifyCircuit < BaseChatNotification + def title + 'Unify Circuit' + end + + def description + s_('Integrations|Send notifications about project events to Unify Circuit.') + end + + def self.to_param + 'unify_circuit' + end + + def help + 'This service sends notifications about projects events to a Unify Circuit conversation.<br /> + To set up this service: + <ol> + <li><a href="https://www.circuit.com/unifyportalfaqdetail?articleId=164448">Set up an incoming webhook for your conversation</a>. All notifications will come to this conversation.</li> + <li>Paste the <strong>Webhook URL</strong> into the field below.</li> + <li>Select events below to enable notifications.</li> + </ol>' + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "e.g. https://circuit.com/rest/v2/webhooks/incoming/…", required: true }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + response = Gitlab::HTTP.post(webhook, body: { + subject: message.project_name, + text: message.summary, + markdown: true + }.to_json) + + response if response.success? + end + + def custom_data(data) + super(data).merge(markdown: true) + end + end +end diff --git a/app/models/integrations/webex_teams.rb b/app/models/integrations/webex_teams.rb new file mode 100644 index 00000000000..3f420331035 --- /dev/null +++ b/app/models/integrations/webex_teams.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Integrations + class WebexTeams < BaseChatNotification + include ActionView::Helpers::UrlHelper + + def title + s_("WebexTeamsService|Webex Teams") + end + + def description + s_("WebexTeamsService|Send notifications about project events to Webex Teams.") + end + + def self.to_param + 'webex_teams' + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/webex_teams'), target: '_blank', rel: 'noopener noreferrer' + s_("WebexTeamsService|Send notifications about project events to a Webex Teams conversation. %{docs_link}") % { docs_link: docs_link.html_safe } + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "https://api.ciscospark.com/v1/webhooks/incoming/...", required: true }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + header = { 'Content-Type' => 'application/json' } + response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.summary }.to_json) + + response if response.success? + end + + def custom_data(data) + super(data).merge(markdown: true) + end + end +end diff --git a/app/models/integrations/youtrack.rb b/app/models/integrations/youtrack.rb new file mode 100644 index 00000000000..10531717f11 --- /dev/null +++ b/app/models/integrations/youtrack.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Integrations + class Youtrack < BaseIssueTracker + include ActionView::Helpers::UrlHelper + + validates :project_url, :issues_url, presence: true, public_url: true, if: :activated? + + # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030 + def self.reference_pattern(only_long: false) + if only_long + /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)/ + else + /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)|(#{Issue.reference_prefix}#{Gitlab::Regex.issue})/ + end + end + + def title + 'YouTrack' + end + + def description + s_("IssueTracker|Use YouTrack as this project's issue tracker.") + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/youtrack'), target: '_blank', rel: 'noopener noreferrer' + s_("IssueTracker|Use YouTrack as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } + end + + def self.to_param + 'youtrack' + end + + def fields + [ + { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in YouTrack.'), required: true }, + { type: 'text', name: 'issues_url', title: s_('ProjectService|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the YouTrack project. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true } + ] + end + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 2077f9bfdbb..b0a126c4442 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -464,6 +464,10 @@ class Issue < ApplicationRecord issue_type_supports?(:assignee) end + def supports_time_tracking? + issue_type_supports?(:time_tracking) + end + def email_participants_emails issue_email_participants.pluck(:email) end @@ -524,7 +528,7 @@ class Issue < ApplicationRecord def could_not_move(exception) # Symptom of running out of space - schedule rebalancing - IssueRebalancingWorker.perform_async(nil, project_id) + IssueRebalancingWorker.perform_async(nil, *project.self_or_root_group_ids) end end diff --git a/app/models/key.rb b/app/models/key.rb index 15b3c460b52..64385953865 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -7,6 +7,7 @@ class Key < ApplicationRecord include Sortable include Sha256Attribute include Expirable + include FromUnion sha256_attribute :fingerprint_sha256 @@ -43,7 +44,9 @@ class Key < ApplicationRecord scope :preload_users, -> { preload(:user) } scope :for_user, -> (user) { where(user: user) } scope :order_last_used_at_desc, -> { reorder(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } - scope :expired_today_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') = CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } + + # Date is set specifically in this scope to improve query time. + scope :expired_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') BETWEEN '2000-01-01' AND CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } scope :expiring_soon_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') > CURRENT_DATE AND date(expires_at AT TIME ZONE 'UTC') < ? AND before_expiry_notification_delivered_at IS NULL", DAYS_TO_EXPIRE.days.from_now.to_date]) } def self.regular_keys diff --git a/app/models/label.rb b/app/models/label.rb index a46d6bc5c0f..1a07620f944 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -9,6 +9,10 @@ class Label < ApplicationRecord include Sortable include FromUnion include Presentable + include IgnorableColumns + + # TODO: Project#create_labels can remove column exception when this column is dropped from all envs + ignore_column :remove_on_close, remove_with: '14.1', remove_after: '2021-06-22' cache_markdown_field :description, pipeline: :single_line diff --git a/app/models/label_link.rb b/app/models/label_link.rb index a466fe69300..4fb5fd8c58a 100644 --- a/app/models/label_link.rb +++ b/app/models/label_link.rb @@ -11,5 +11,4 @@ class LabelLink < ApplicationRecord validates :label, presence: true, unless: :importing? scope :for_target, -> (target_id, target_type) { where(target_id: target_id, target_type: target_type) } - scope :with_remove_on_close_labels, -> { joins(:label).where(labels: { remove_on_close: true }) } end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b837b902e2d..53e7d52c558 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -7,7 +7,7 @@ class LfsObject < ApplicationRecord include ObjectStorage::BackgroundMove include FileStoreMounter - has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :lfs_objects_projects has_many :projects, -> { distinct }, through: :lfs_objects_projects scope :with_files_stored_locally, -> { where(file_store: LfsObjectUploader::Store::LOCAL) } @@ -18,6 +18,8 @@ class LfsObject < ApplicationRecord mount_file_store_uploader LfsObjectUploader + BATCH_SIZE = 3000 + def self.not_linked_to_project(project) where('NOT EXISTS (?)', project.lfs_objects_projects.select(1).where('lfs_objects_projects.lfs_object_id = lfs_objects.id')) @@ -37,13 +39,14 @@ class LfsObject < ApplicationRecord file_store == LfsObjectUploader::Store::LOCAL end - # rubocop: disable Cop/DestroyAll - def self.destroy_unreferenced - joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") - .where(lfs_objects_projects: { id: nil }) - .destroy_all + def self.unreferenced_in_batches + each_batch(of: BATCH_SIZE, order: :desc) do |lfs_objects| + relation = lfs_objects.where('NOT EXISTS (?)', + LfsObjectsProject.select(1).where('lfs_objects_projects.lfs_object_id = lfs_objects.id')) + + yield relation if relation.any? + end end - # rubocop: enable Cop/DestroyAll def self.calculate_oid(path) self.hexdigest(path) diff --git a/app/models/member.rb b/app/models/member.rb index 044b662e10f..0636c3c2d4e 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -14,6 +14,7 @@ class Member < ApplicationRecord include UpdateHighestRole AVATAR_SIZE = 40 + ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 attr_accessor :raw_invite_token @@ -107,10 +108,14 @@ class Member < ApplicationRecord scope :active_without_invites_and_requests, -> do left_join_users .where(users: { state: 'active' }) - .non_request + .without_invites_and_requests + .reorder(nil) + end + + scope :without_invites_and_requests, -> do + non_request .non_invite .non_minimal_access - .reorder(nil) end scope :invite, -> { where.not(invite_token: nil) } @@ -166,10 +171,10 @@ class Member < ApplicationRecord after_create :send_invite, if: :invite?, unless: :importing? after_create :send_request, if: :request?, unless: :importing? after_create :create_notification_setting, unless: [:pending?, :importing?] - after_create :post_create_hook, unless: [:pending?, :importing?] - after_update :post_update_hook, unless: [:pending?, :importing?] + after_create :post_create_hook, unless: [:pending?, :importing?], if: :hook_prerequisites_met? + after_update :post_update_hook, unless: [:pending?, :importing?], if: :hook_prerequisites_met? after_destroy :destroy_notification_setting - after_destroy :post_destroy_hook, unless: :pending? + after_destroy :post_destroy_hook, unless: :pending?, if: :hook_prerequisites_met? after_commit :refresh_member_authorized_projects default_value_for :notification_level, NotificationSetting.levels[:global] @@ -336,7 +341,7 @@ class Member < ApplicationRecord return User.find_by(id: user) if user.is_a?(Integer) - User.find_by(email: user) || user + User.find_by_any_email(user) || user end def retrieve_member(source, user, existing_members) @@ -383,6 +388,12 @@ class Member < ApplicationRecord invite? || request? end + def hook_prerequisites_met? + # It is essential that an associated user record exists + # so that we can successfully fire any member related hooks/notifications. + user.present? + end + def accept_request return false unless request? diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index b22a4fa9ef6..c7bc31cde5d 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -8,7 +8,7 @@ class GroupMember < Member belongs_to :group, foreign_key: 'source_id' alias_attribute :namespace_id, :source_id - delegate :update_two_factor_requirement, to: :user + delegate :update_two_factor_requirement, to: :user, allow_nil: true # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE @@ -36,6 +36,10 @@ class GroupMember < Member Gitlab::Access.sym_options_with_owner end + def self.pluck_user_ids + pluck(:user_id) + end + def group source end diff --git a/app/models/members/last_group_owner_assigner.rb b/app/models/members/last_group_owner_assigner.rb index 64decb1df36..dcf0a2d0ad3 100644 --- a/app/models/members/last_group_owner_assigner.rb +++ b/app/models/members/last_group_owner_assigner.rb @@ -1,46 +1,44 @@ # frozen_string_literal: true -module Members - class LastGroupOwnerAssigner - def initialize(group, members) - @group = group - @members = members - end +class LastGroupOwnerAssigner + def initialize(group, members) + @group = group + @members = members + end - def execute - @last_blocked_owner = no_owners_in_heirarchy? && group.single_blocked_owner? - @group_single_owner = owners.size == 1 + def execute + @last_blocked_owner = no_owners_in_heirarchy? && group.single_blocked_owner? + @group_single_owner = owners.size == 1 - members.each { |member| set_last_owner(member) } - end + members.each { |member| set_last_owner(member) } + end - private + private - attr_reader :group, :members, :last_blocked_owner, :group_single_owner + attr_reader :group, :members, :last_blocked_owner, :group_single_owner - def no_owners_in_heirarchy? - owners.empty? - end + def no_owners_in_heirarchy? + owners.empty? + end - def set_last_owner(member) - member.last_owner = member.id.in?(owner_ids) && group_single_owner - member.last_blocked_owner = member.id.in?(blocked_owner_ids) && last_blocked_owner - end + def set_last_owner(member) + member.last_owner = member.id.in?(owner_ids) && group_single_owner + member.last_blocked_owner = member.id.in?(blocked_owner_ids) && last_blocked_owner + end - def owner_ids - @owner_ids ||= owners.where(id: member_ids).ids - end + def owner_ids + @owner_ids ||= owners.where(id: member_ids).ids + end - def blocked_owner_ids - @blocked_owner_ids ||= group.blocked_owners.where(id: member_ids).ids - end + def blocked_owner_ids + @blocked_owner_ids ||= group.blocked_owners.where(id: member_ids).ids + end - def member_ids - @members_ids ||= members.pluck(:id) - end + def member_ids + @members_ids ||= members.pluck(:id) + end - def owners - @owners ||= group.members_with_parents.owners.load - end + def owners + @owners ||= group.members_with_parents.owners.load end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index aaef56418d2..15f112690d5 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -37,7 +37,7 @@ class MergeRequest < ApplicationRecord SORTING_PREFERENCE_FIELD = :merge_requests_sort ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON = { - 'Ci::CompareMetricsReportsService' => ->(project) { ::Gitlab::Ci::Features.merge_base_pipeline_for_metrics_comparison?(project) }, + 'Ci::CompareMetricsReportsService' => ->(project) { true }, 'Ci::CompareCodequalityReportsService' => ->(project) { true } }.freeze @@ -125,6 +125,8 @@ class MergeRequest < ApplicationRecord ].freeze serialize :merge_params, Hash # rubocop:disable Cop/ActiveRecordSerialize + before_validation :set_draft_status + after_create :ensure_merge_request_diff after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed @@ -267,6 +269,7 @@ class MergeRequest < ApplicationRecord scope :merged, -> { with_state(:merged) } scope :closed_and_merged, -> { with_states(:closed, :merged) } scope :open_and_closed, -> { with_states(:opened, :closed) } + scope :drafts, -> { where(draft: true) } scope :from_source_branches, ->(branches) { where(source_branch: branches) } scope :by_commit_sha, ->(sha) do where('EXISTS (?)', MergeRequestDiff.select(1).where('merge_requests.latest_merge_request_diff_id = merge_request_diffs.id').by_commit_sha(sha)).reorder(nil) @@ -1908,6 +1911,10 @@ class MergeRequest < ApplicationRecord private + def set_draft_status + self.draft = draft? + end + def missing_report_error(report_type) { status: :error, status_reason: "This merge request does not have #{report_type} reports" } end diff --git a/app/models/merge_request_context_commit.rb b/app/models/merge_request_context_commit.rb index e081a96dc10..0f2a7515462 100644 --- a/app/models/merge_request_context_commit.rb +++ b/app/models/merge_request_context_commit.rb @@ -12,7 +12,7 @@ class MergeRequestContextCommit < ApplicationRecord validates :sha, presence: true validates :sha, uniqueness: { message: 'has already been added' } - serialize :trailers, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize + serialize :trailers, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize validates :trailers, json_schema: { filename: 'git_trailers' } # Sort by committed date in descending order to ensure latest commits comes on the top diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 2dc6796732f..f58d7788432 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -389,11 +389,23 @@ class MergeRequestDiff < ApplicationRecord def diffs_in_batch(batch_page, batch_size, diff_options:) fetching_repository_diffs(diff_options) do |comparison| + reorder_diff_files! + diffs_batch = diffs_in_batch_collection(batch_page, batch_size, diff_options: diff_options) + if comparison - comparison.diffs_in_batch(batch_page, batch_size, diff_options: diff_options) + if diff_options[:paths].blank? && !without_files? + # Return the empty MergeRequestDiffBatch for an out of bound batch request + break diffs_batch if diffs_batch.diff_file_paths.blank? + + diff_options.merge!( + paths: diffs_batch.diff_file_paths, + pagination_data: diffs_batch.pagination_data + ) + end + + comparison.diffs(diff_options) else - reorder_diff_files! - diffs_in_batch_collection(batch_page, batch_size, diff_options: diff_options) + diffs_batch end end end diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index 259690ef308..ed398e0d2e0 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -12,7 +12,7 @@ class MergeRequestDiffCommit < ApplicationRecord sha_attribute :sha alias_attribute :id, :sha - serialize :trailers, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize + serialize :trailers, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize validates :trailers, json_schema: { filename: 'git_trailers' } # Deprecated; use `bulk_insert!` from `BulkInsertSafe` mixin instead. diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 16090f0ebfa..9ed6c106e45 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -36,6 +36,7 @@ class Milestone < ApplicationRecord scope :order_by_dates_and_title, -> { order(due_date: :asc, start_date: :asc, title: :asc) } validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } + validate :uniqueness_of_title, if: :title_changed? state_machine :state, initial: :active do event :close do @@ -172,4 +173,16 @@ class Milestone < ApplicationRecord def issues_finder_params { project_id: project_id, group_id: group_id, include_subgroups: group_id.present? }.compact end + + # milestone titles must be unique across project and group milestones + def uniqueness_of_title + if project + relation = self.class.for_projects_and_groups([project_id], [project.group&.id]) + elsif group + relation = self.class.for_projects_and_groups(group.projects.select(:id), [group.id]) + end + + title_exists = relation.find_by_title(title) + errors.add(:title, _("already being used for another group or project %{timebox_name}.") % { timebox_name: timebox_name }) if title_exists + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 8f03c6145cb..90e06e44165 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -271,14 +271,9 @@ class Namespace < ApplicationRecord # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects - namespace = user? ? self : self_and_descendants - Project.where(namespace: namespace) - end + namespace = user? ? self : self_and_descendant_ids - # Includes pipelines from this namespace and pipelines from all subgroups - # that belongs to this namespace - def all_pipelines - Ci::Pipeline.where(project: all_projects) + Project.where(namespace: namespace) end def has_parent? @@ -442,12 +437,6 @@ class Namespace < ApplicationRecord end def all_projects_with_pages - if all_projects.pages_metadata_not_migrated.exists? - Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation( - all_projects.pages_metadata_not_migrated - ) - end - all_projects.with_pages_deployed end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 75b8169b58e..600abc33471 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -14,7 +14,8 @@ class NamespaceSetting < ApplicationRecord before_validation :normalize_default_branch_name NAMESPACE_SETTINGS_PARAMS = [:default_branch_name, :delayed_project_removal, - :lock_delayed_project_removal, :resource_access_token_creation_allowed].freeze + :lock_delayed_project_removal, :resource_access_token_creation_allowed, + :prevent_sharing_groups_outside_hierarchy].freeze self.primary_key = :namespace_id diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index a1711bc5ee0..d0281f4d974 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -46,6 +46,12 @@ module Namespaces after_update :sync_traversal_ids, if: -> { sync_traversal_ids? && saved_change_to_parent_id? } scope :traversal_ids_contains, ->(ids) { where("traversal_ids @> (?)", ids) } + # When filtering namespaces by the traversal_ids column to compile a + # list of namespace IDs, it's much faster to reference the ID in + # traversal_ids than the primary key ID column. + # WARNING This scope must be used behind a linear query feature flag + # such as `use_traversal_ids`. + scope :as_ids, -> { select('traversal_ids[array_length(traversal_ids, 1)] AS id') } end def sync_traversal_ids? @@ -58,12 +64,30 @@ module Namespaces traversal_ids.present? end + def root_ancestor + return super if parent.nil? + return super unless persisted? + + return super if traversal_ids.blank? + return super unless Feature.enabled?(:use_traversal_ids_for_root_ancestor, default_enabled: :yaml) + + strong_memoize(:root_ancestor) do + Namespace.find_by(id: traversal_ids.first) + end + end + def self_and_descendants return super unless use_traversal_ids? lineage(top: self) end + def self_and_descendant_ids + return super unless use_traversal_ids? + + self_and_descendants.as_ids + end + def descendants return super unless use_traversal_ids? @@ -88,7 +112,8 @@ module Namespaces # Clear any previously memoized root_ancestor as our ancestors have changed. clear_memoization(:root_ancestor) - Namespace::TraversalHierarchy.for_namespace(root_ancestor).sync_traversal_ids! + # We cannot rely on Namespaces::Traversal::Linear#root_ancestor because it might be stale + Namespace::TraversalHierarchy.for_namespace(recursive_root_ancestor).sync_traversal_ids! end # Lock the root of the hierarchy we just left, and lock the root of the hierarchy diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index 409438f53d2..5a1a9d24117 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -16,6 +16,7 @@ module Namespaces parent.root_ancestor end end + alias_method :recursive_root_ancestor, :root_ancestor # Returns all ancestors, self, and descendants of the current namespace. def self_and_hierarchy @@ -61,6 +62,11 @@ module Namespaces end alias_method :recursive_self_and_descendants, :self_and_descendants + def self_and_descendant_ids + self_and_descendants.select(:id) + end + alias_method :recursive_self_and_descendant_ids, :self_and_descendant_ids + def object_hierarchy(ancestors_base) Gitlab::ObjectHierarchy.new(ancestors_base, options: { use_distinct: Feature.enabled?(:use_distinct_in_object_hierarchy, self) }) end diff --git a/app/models/note.rb b/app/models/note.rb index ae4a8859d4d..d1a59394ba1 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -96,7 +96,9 @@ class Note < ApplicationRecord validate :does_not_exceed_notes_limit?, on: :create, unless: [:system?, :importing?] - # @deprecated attachments are handler by the MarkdownUploader + # @deprecated attachments are handled by the Upload model. + # + # https://gitlab.com/gitlab-org/gitlab/-/issues/20830 mount_uploader :attachment, AttachmentUploader # Scopes @@ -274,6 +276,10 @@ class Note < ApplicationRecord noteable_type == 'AlertManagement::Alert' end + def for_vulnerability? + noteable_type == "Vulnerability" + end + def for_project_snippet? noteable.is_a?(ProjectSnippet) end @@ -409,6 +415,8 @@ class Note < ApplicationRecord 'snippet' elsif for_alert_mangement_alert? 'alert_management_alert' + elsif for_vulnerability? + 'security_resource' else noteable_type.demodulize.underscore end diff --git a/app/models/onboarding_progress.rb b/app/models/onboarding_progress.rb index be76c3dbf9d..9185547d7cd 100644 --- a/app/models/onboarding_progress.rb +++ b/app/models/onboarding_progress.rb @@ -85,6 +85,10 @@ class OnboardingProgress < ApplicationRecord end end + def number_of_completed_actions + attributes.extract!(*ACTIONS.map { |action| self.class.column_name(action).to_s }).compact!.size + end + private def namespace_is_root_namespace diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 537543a7ff0..8b052f80395 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -49,6 +49,8 @@ module Operations scope :enabled, -> { where(active: true) } scope :disabled, -> { where(active: false) } + scope :new_version_only, -> { where(version: :new_version_flag)} + enum version: { legacy_flag: 1, new_version_flag: 2 diff --git a/app/models/packages/debian/group_distribution_key.rb b/app/models/packages/debian/group_distribution_key.rb new file mode 100644 index 00000000000..a60ddca32e2 --- /dev/null +++ b/app/models/packages/debian/group_distribution_key.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Packages::Debian::GroupDistributionKey < ApplicationRecord + def self.container_type + :group + end + + include Packages::Debian::DistributionKey +end diff --git a/app/models/packages/debian/project_distribution_key.rb b/app/models/packages/debian/project_distribution_key.rb new file mode 100644 index 00000000000..69cf2791b02 --- /dev/null +++ b/app/models/packages/debian/project_distribution_key.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Packages::Debian::ProjectDistributionKey < ApplicationRecord + def self.container_type + :project + end + + include Packages::Debian::DistributionKey +end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 36edf646658..7b0bb72940e 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -8,6 +8,23 @@ class Packages::Package < ApplicationRecord DISPLAYABLE_STATUSES = [:default, :error].freeze INSTALLABLE_STATUSES = [:default].freeze + enum package_type: { + maven: 1, + npm: 2, + conan: 3, + nuget: 4, + pypi: 5, + composer: 6, + generic: 7, + golang: 8, + debian: 9, + rubygems: 10, + helm: 11, + terraform_module: 12 + } + + enum status: { default: 0, hidden: 1, processing: 2, error: 3 } + belongs_to :project belongs_to :creator, class_name: 'User' @@ -59,7 +76,7 @@ class Packages::Package < ApplicationRecord validates :version, format: { with: Gitlab::Regex.maven_version_regex }, if: -> { version? && maven? } validates :version, format: { with: Gitlab::Regex.pypi_version_regex }, if: :pypi? validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :golang? - validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :helm? + validates :version, format: { with: Gitlab::Regex.helm_version_regex }, if: :helm? validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? || terraform_module? } validates :version, @@ -72,12 +89,6 @@ class Packages::Package < ApplicationRecord if: :debian_package? validate :forbidden_debian_changes, if: :debian? - enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, - composer: 6, generic: 7, golang: 8, debian: 9, - rubygems: 10, helm: 11, terraform_module: 12 } - - enum status: { default: 0, hidden: 1, processing: 2, error: 3 } - scope :for_projects, ->(project_ids) { where(project_id: project_ids) } scope :with_name, ->(name) { where(name: name) } scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } @@ -133,14 +144,24 @@ class Packages::Package < ApplicationRecord scope :order_type_desc, -> { reorder(package_type: :desc) } scope :order_project_name, -> { joins(:project).reorder('projects.name ASC') } scope :order_project_name_desc, -> { joins(:project).reorder('projects.name DESC') } - scope :order_project_path, -> { joins(:project).reorder('projects.path ASC, id ASC') } - scope :order_project_path_desc, -> { joins(:project).reorder('projects.path DESC, id DESC') } scope :order_by_package_file, -> { joins(:package_files).order('packages_package_files.created_at ASC') } + scope :order_project_path, -> do + keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :asc) + + joins(:project).reorder(keyset_order) + end + + scope :order_project_path_desc, -> do + keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :desc) + + joins(:project).reorder(keyset_order) + end + after_commit :update_composer_cache, on: :destroy, if: -> { composer? } def self.only_maven_packages_with_path(path, use_cte: false) - if use_cte && Feature.enabled?(:maven_metadata_by_path_with_optimization_fence, default_enabled: :yaml) + if use_cte # This is an optimization fence which assumes that looking up the Metadatum record by path (globally) # and then filter down the packages (by project or by group and subgroups) will be cheaper than # looking up all packages within a project or group and filter them by path. @@ -196,6 +217,32 @@ class Packages::Package < ApplicationRecord end end + def self.keyset_pagination_order(join_class:, column_name:, direction: :asc) + join_table = join_class.table_name + asc_order_expression = Gitlab::Database.nulls_last_order("#{join_table}.#{column_name}", :asc) + desc_order_expression = Gitlab::Database.nulls_first_order("#{join_table}.#{column_name}", :desc) + order_direction = direction == :asc ? asc_order_expression : desc_order_expression + reverse_order_direction = direction == :asc ? desc_order_expression : asc_order_expression + arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert + + ::Gitlab::Pagination::Keyset::Order.build([ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: "#{join_table}_#{column_name}", + column_expression: join_class.arel_table[column_name], + order_expression: order_direction, + reversed_order_expression: reverse_order_direction, + order_direction: direction, + distinct: false, + add_to_projections: true + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: arel_order_classes[direction].new(Packages::Package.arel_table[:id]), + add_to_projections: true + ) + ]) + end + def versions project.packages .including_build_info @@ -222,6 +269,10 @@ class Packages::Package < ApplicationRecord tags.pluck(:name) end + def infrastructure_package? + terraform_module? + end + def debian_incoming? debian? && version.nil? end diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 3d8641ca2fa..3ef30c035e8 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -33,11 +33,18 @@ class Packages::PackageFile < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) } scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) } scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) } + scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) } scope :for_rubygem_with_file_name, ->(project, file_name) do joins(:package).merge(project.packages.rubygems).with_file_name(file_name) end + scope :for_helm_with_channel, ->(project, channel) do + joins(:package).merge(project.packages.helm.installable) + .joins(:helm_file_metadatum) + .where(packages_helm_file_metadata: { channel: channel }) + end + scope :with_conan_file_type, ->(file_type) do joins(:conan_file_metadatum) .where(packages_conan_file_metadata: { conan_file_type: ::Packages::Conan::FileMetadatum.conan_file_types[file_type] }) diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index 17131cd736d..e7d455085c0 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -26,7 +26,18 @@ module Pages end def source - zip_source || legacy_source + return unless deployment&.file + + global_id = ::Gitlab::GlobalId.build(deployment, id: deployment.id).to_s + + { + type: 'zip', + path: deployment.file.url_or_file_path(expire_at: 1.day.from_now), + global_id: global_id, + sha256: deployment.file_sha256, + file_size: deployment.size, + file_count: deployment.file_count + } end def prefix @@ -46,32 +57,5 @@ module Pages project.pages_metadatum.pages_deployment end end - - def zip_source - return unless deployment&.file - - global_id = ::Gitlab::GlobalId.build(deployment, id: deployment.id).to_s - - { - type: 'zip', - path: deployment.file.url_or_file_path(expire_at: 1.day.from_now), - global_id: global_id, - sha256: deployment.file_sha256, - file_size: deployment.size, - file_count: deployment.file_count - } - end - - # TODO: remove support for legacy storage in 14.3 https://gitlab.com/gitlab-org/gitlab/-/issues/328712 - # we support this till 14.3 to allow people to still use legacy storage if something goes very wrong - # on self-hosted installations, and we'll need some time to fix it - def legacy_source - return unless ::Settings.pages.local_store.enabled - - { - type: 'file', - path: File.join(project.full_path, 'public/') - } - end end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 4668fc265a0..c932d0bf800 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -50,6 +50,8 @@ class PagesDomain < ApplicationRecord after_update :update_daemon, if: :saved_change_to_pages_config? after_destroy :update_daemon + scope :for_project, ->(project) { where(project: project) } + scope :enabled, -> { where('enabled_until >= ?', Time.current ) } scope :needs_verification, -> do verified_at = arel_table[:verified_at] @@ -225,16 +227,6 @@ class PagesDomain < ApplicationRecord def pages_deployed? return false unless project - # TODO: remove once `pages_metadatum` is migrated - # https://gitlab.com/gitlab-org/gitlab/issues/33106 - unless project.pages_metadatum - Gitlab::BackgroundMigration::MigratePagesMetadata - .new - .perform_on_relation(Project.where(id: project_id)) - - project.reset - end - project.pages_metadatum&.deployed? end diff --git a/app/models/postgresql/replication_slot.rb b/app/models/postgresql/replication_slot.rb index c96786423e5..77b42c34ad9 100644 --- a/app/models/postgresql/replication_slot.rb +++ b/app/models/postgresql/replication_slot.rb @@ -26,8 +26,8 @@ module Postgresql "(pg_current_wal_insert_lsn(), restart_lsn)::bigint" # We force the use of a transaction here so the query always goes to the - # primary, even when using the EE DB load balancer. - sizes = transaction { pluck(lag_function) } + # primary, even when using the DB load balancer. + sizes = transaction { pluck(Arel.sql(lag_function)) } too_great = sizes.compact.count { |size| size >= max } # If too many replicas are falling behind too much, the availability of a diff --git a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb index 671091480ee..c0ed56057ae 100644 --- a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Preloaders - # This class preloads the max access level for the user within the given projects and + # This class preloads the max access level (role) for the user within the given projects and # stores the values in requests store via the ProjectTeam class. class UserMaxAccessLevelInProjectsPreloader def initialize(projects, user) diff --git a/app/models/project.rb b/app/models/project.rb index 9d572b7e2f8..735dc185575 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -63,8 +63,6 @@ class Project < ApplicationRecord VALID_MIRROR_PORTS = [22, 80, 443].freeze VALID_MIRROR_PROTOCOLS = %w(http https ssh git).freeze - ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 - SORTING_PREFERENCE_FIELD = :projects_sort MAX_BUILD_TIMEOUT = 1.month @@ -129,40 +127,6 @@ class Project < ApplicationRecord after_create :check_repository_absence! acts_as_ordered_taggable_on :topics - # The 'tag_list' alias and the 'has_many' associations are required during the 'tags -> topics' migration - # TODO: eliminate 'tag_list', 'topic_taggings' and 'tags' in the further process of the migration - # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 - alias_attribute :tag_list, :topic_list - has_many :topic_taggings, -> { includes(:tag).order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, - as: :taggable, - class_name: 'ActsAsTaggableOn::Tagging', - after_add: :dirtify_tag_list, - after_remove: :dirtify_tag_list - has_many :topics, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, - class_name: 'ActsAsTaggableOn::Tag', - through: :topic_taggings, - source: :tag - has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, - class_name: 'ActsAsTaggableOn::Tag', - through: :topic_taggings, - source: :tag - - # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1]. - # [1] https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61237 - # TODO: remove 'topic_list' and 'topic_list=' once the background migration is complete - # https://gitlab.com/gitlab-org/gitlab/-/issues/331081 - def topic_list - # Return both old topics (context 'tags') and new topics (context 'topics') - tag_list_on('tags') + tag_list_on('topics') - end - - def topic_list=(new_tags) - # Old topics with context 'tags' are added as new topics with context 'topics' - super(new_tags) - - # Remove old topics with context 'tags' - set_tag_list_on('tags', '') - end attr_accessor :old_path_with_namespace attr_accessor :template_name @@ -182,44 +146,51 @@ class Project < ApplicationRecord has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' has_many :boards + def self.integration_association_name(name) + if ::Integration.renamed?(name) + "#{name}_integration" + else + "#{name}_service" + end + end + # Project integrations - has_one :asana_service, class_name: 'Integrations::Asana' - has_one :assembla_service, class_name: 'Integrations::Assembla' - has_one :bamboo_service, class_name: 'Integrations::Bamboo' - has_one :campfire_service, class_name: 'Integrations::Campfire' - has_one :confluence_service, class_name: 'Integrations::Confluence' - has_one :datadog_service, class_name: 'Integrations::Datadog' + has_one :asana_integration, class_name: 'Integrations::Asana' + has_one :assembla_integration, class_name: 'Integrations::Assembla' + has_one :bamboo_integration, class_name: 'Integrations::Bamboo' + has_one :bugzilla_integration, class_name: 'Integrations::Bugzilla' + has_one :buildkite_integration, class_name: 'Integrations::Buildkite' + has_one :campfire_integration, class_name: 'Integrations::Campfire' + has_one :confluence_integration, class_name: 'Integrations::Confluence' + has_one :custom_issue_tracker_integration, class_name: 'Integrations::CustomIssueTracker' + has_one :datadog_integration, class_name: 'Integrations::Datadog' + has_one :discord_integration, class_name: 'Integrations::Discord' + has_one :drone_ci_integration, class_name: 'Integrations::DroneCi' has_one :emails_on_push_service, class_name: 'Integrations::EmailsOnPush' - has_one :discord_service - has_one :drone_ci_service - has_one :ewm_service - has_one :pipelines_email_service - has_one :irker_service - has_one :pivotaltracker_service - has_one :flowdock_service - has_one :mattermost_slash_commands_service - has_one :mattermost_service - has_one :slack_slash_commands_service - has_one :slack_service - has_one :buildkite_service - has_one :teamcity_service - has_one :pushover_service - has_one :jenkins_service - has_one :jira_service - has_one :redmine_service - has_one :youtrack_service - has_one :custom_issue_tracker_service - has_one :bugzilla_service - has_one :external_wiki_service + has_one :ewm_service, class_name: 'Integrations::Ewm' + has_one :external_wiki_service, class_name: 'Integrations::ExternalWiki' + has_one :flowdock_service, class_name: 'Integrations::Flowdock' + has_one :hangouts_chat_service, class_name: 'Integrations::HangoutsChat' + has_one :irker_service, class_name: 'Integrations::Irker' + has_one :jenkins_service, class_name: 'Integrations::Jenkins' + has_one :jira_service, class_name: 'Integrations::Jira' + has_one :mattermost_service, class_name: 'Integrations::Mattermost' + has_one :mattermost_slash_commands_service, class_name: 'Integrations::MattermostSlashCommands' + has_one :microsoft_teams_service, class_name: 'Integrations::MicrosoftTeams' + has_one :mock_ci_service, class_name: 'Integrations::MockCi' + has_one :packagist_service, class_name: 'Integrations::Packagist' + has_one :pipelines_email_service, class_name: 'Integrations::PipelinesEmail' + has_one :pivotaltracker_service, class_name: 'Integrations::Pivotaltracker' + has_one :pushover_service, class_name: 'Integrations::Pushover' + has_one :redmine_service, class_name: 'Integrations::Redmine' + has_one :slack_service, class_name: 'Integrations::Slack' + has_one :slack_slash_commands_service, class_name: 'Integrations::SlackSlashCommands' + has_one :teamcity_service, class_name: 'Integrations::Teamcity' + has_one :unify_circuit_service, class_name: 'Integrations::UnifyCircuit' + has_one :webex_teams_service, class_name: 'Integrations::WebexTeams' + has_one :youtrack_service, class_name: 'Integrations::Youtrack' has_one :prometheus_service, inverse_of: :project - has_one :mock_ci_service - has_one :mock_deployment_service has_one :mock_monitoring_service - has_one :microsoft_teams_service - has_one :packagist_service - has_one :hangouts_chat_service - has_one :unify_circuit_service - has_one :webex_teams_service has_one :root_of_fork_network, foreign_key: 'root_project_id', @@ -261,7 +232,15 @@ class Project < ApplicationRecord has_many :events has_many :milestones has_many :iterations - has_many :notes + + # Projects with a very large number of notes may time out destroying them + # through the foreign key. Additionally, the deprecated attachment uploader + # for notes requires us to use dependent: :destroy to avoid orphaning uploaded + # files. + # + # https://gitlab.com/gitlab-org/gitlab/-/issues/207222 + has_many :notes, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :snippets, class_name: 'ProjectSnippet' has_many :hooks, class_name: 'ProjectHook' has_many :protected_branches @@ -287,7 +266,7 @@ class Project < ApplicationRecord has_many :users_star_projects has_many :starrers, through: :users_star_projects, source: :user has_many :releases - has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :lfs_objects_projects has_many :lfs_objects, -> { distinct }, through: :lfs_objects_projects has_many :lfs_file_locks has_many :project_group_links @@ -439,6 +418,7 @@ class Project < ApplicationRecord delegate :scheduled?, :started?, :in_progress?, :failed?, :finished?, prefix: :import, to: :import_state, allow_nil: true delegate :squash_always?, :squash_never?, :squash_enabled_by_default?, :squash_readonly?, to: :project_setting + delegate :squash_option, to: :project_setting delegate :no_import?, to: :import_state, allow_nil: true delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true @@ -449,11 +429,12 @@ class Project < ApplicationRecord delegate :last_pipeline, to: :commit, allow_nil: true delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true delegate :dashboard_timezone, to: :metrics_setting, allow_nil: true, prefix: true - delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci - delegate :forward_deployment_enabled, :forward_deployment_enabled=, :forward_deployment_enabled?, to: :ci_cd_settings, prefix: :ci - delegate :keep_latest_artifact, :keep_latest_artifact=, :keep_latest_artifact?, :keep_latest_artifacts_available?, to: :ci_cd_settings + delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :forward_deployment_enabled, :forward_deployment_enabled=, :forward_deployment_enabled?, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :job_token_scope_enabled, :job_token_scope_enabled=, :job_token_scope_enabled?, to: :ci_cd_settings, prefix: :ci + delegate :keep_latest_artifact, :keep_latest_artifact=, :keep_latest_artifact?, :keep_latest_artifacts_available?, to: :ci_cd_settings, allow_nil: true delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, :restrict_user_defined_variables?, - to: :ci_cd_settings + to: :ci_cd_settings, allow_nil: true delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, :allow_merge_on_skipped_pipeline=, :has_confluence?, :allow_editing_commit_messages?, @@ -561,7 +542,7 @@ class Project < ApplicationRecord scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).merge(Event.pushed_action) } scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } - scope :with_active_jira_services, -> { joins(:integrations).merge(::JiraService.active) } # rubocop:disable CodeReuse/ServiceClass + scope :with_active_jira_services, -> { joins(:integrations).merge(::Integrations::Jira.active) } scope :with_jira_dvcs_cloud, -> { joins(:feature_usage).merge(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: true)) } scope :with_jira_dvcs_server, -> { joins(:feature_usage).merge(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } scope :inc_routes, -> { includes(:route, namespace: :route) } @@ -637,6 +618,12 @@ class Project < ApplicationRecord scope :with_tracing_enabled, -> { joins(:tracing_setting) } scope :with_enabled_error_tracking, -> { joins(:error_tracking_setting).where(project_error_tracking_settings: { enabled: true }) } + scope :with_service_desk_key, -> (key) do + # project_key is not indexed for now + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24063#note_282435524 for details + joins(:service_desk_setting).where('service_desk_settings.project_key' => key) + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -652,7 +639,7 @@ class Project < ApplicationRecord mount_uploader :bfg_object_map, AttachmentUploader def self.with_api_entity_associations - preload(:project_feature, :route, :tags, :group, :timelogs, namespace: [:route, :owner]) + preload(:project_feature, :route, :topics, :group, :timelogs, namespace: [:route, :owner]) end def self.with_web_entity_associations @@ -838,12 +825,6 @@ class Project < ApplicationRecord from_union([with_issues_enabled, with_merge_requests_enabled]).select(:id) end - - def find_by_service_desk_project_key(key) - # project_key is not indexed for now - # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24063#note_282435524 for details - joins(:service_desk_setting).find_by('service_desk_settings.project_key' => key) - end end def initialize(attributes = nil) @@ -921,6 +902,10 @@ class Project < ApplicationRecord alias_method :ancestors, :ancestors_upto + def ancestors_upto_ids(...) + ancestors_upto(...).pluck(:id) + end + def emails_disabled? strong_memoize(:emails_disabled) do # disabling in the namespace overrides the project setting @@ -1407,9 +1392,9 @@ class Project < ApplicationRecord end def disabled_services - return %w[datadog hipchat] unless Feature.enabled?(:datadog_ci_integration, self) + return %w[datadog] unless Feature.enabled?(:datadog_ci_integration, self) - %w[hipchat] + [] end def find_or_initialize_service(name) @@ -1421,7 +1406,8 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def create_labels Label.templates.each do |label| - params = label.attributes.except('id', 'template', 'created_at', 'updated_at', 'type') + # TODO: remove_on_close exception can be removed after the column is dropped from all envs + params = label.attributes.except('id', 'template', 'created_at', 'updated_at', 'type', 'remove_on_close') Labels::FindOrCreateService.new(nil, self, params).execute(skip_authorization: true) end end @@ -1735,7 +1721,11 @@ class Project < ApplicationRecord end def shared_runners - @shared_runners ||= shared_runners_available? ? Ci::Runner.instance_type : Ci::Runner.none + @shared_runners ||= shared_runners_enabled? ? Ci::Runner.instance_type : Ci::Runner.none + end + + def available_shared_runners + @available_shared_runners ||= shared_runners_available? ? shared_runners : Ci::Runner.none end def group_runners @@ -1746,17 +1736,16 @@ class Project < ApplicationRecord Ci::Runner.from_union([runners, group_runners, shared_runners]) end + def all_available_runners + Ci::Runner.from_union([runners, group_runners, available_shared_runners]) + end + def active_runners strong_memoize(:active_runners) do - all_runners.active + all_available_runners.active end end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/326989 - def any_active_runners?(&block) - active_runners_with_tags.any?(&block) - end - def any_online_runners?(&block) online_runners_with_tags.any?(&block) end @@ -1772,7 +1761,7 @@ class Project < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass # rubocop: disable CodeReuse/ServiceClass - def open_merge_requests_count + def open_merge_requests_count(_current_user = nil) Projects::OpenMergeRequestsCountService.new(self).count end # rubocop: enable CodeReuse/ServiceClass @@ -2006,7 +1995,11 @@ class Project < ApplicationRecord end def export_file_exists? - export_file&.file + import_export_upload&.export_file_exists? + end + + def export_archive_exists? + import_export_upload&.export_archive_exists? end def export_file @@ -2046,7 +2039,6 @@ class Project < ApplicationRecord .append(key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level)) .append(key: 'CI_PROJECT_REPOSITORY_LANGUAGES', value: repository_languages.map(&:name).join(',').downcase) .append(key: 'CI_DEFAULT_BRANCH', value: default_branch) - .append(key: 'CI_PROJECT_CONFIG_PATH', value: ci_config_path_or_default) .append(key: 'CI_CONFIG_PATH', value: ci_config_path_or_default) end @@ -2377,7 +2369,7 @@ class Project < ApplicationRecord end def mark_primary_write_location - # Overriden in EE + ::Gitlab::Database::LoadBalancing::Sticking.mark_primary_write_location(:project, self.id) end def toggle_ci_cd_settings!(settings_attribute) @@ -2454,7 +2446,7 @@ class Project < ApplicationRecord end def access_request_approvers_to_be_notified - members.maintainers.connected_to_user.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) + members.maintainers.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end def pages_lookup_path(trim_prefix: nil, domain: nil) @@ -2562,6 +2554,17 @@ class Project < ApplicationRecord end end + # for projects that are part of user namespace, return project. + def self_or_root_group_ids + if group + root_group = root_namespace + else + project = self + end + + [project&.id, root_group&.id] + end + def package_already_taken?(package_name) namespace.root_ancestor.all_projects .joins(:packages) @@ -2604,10 +2607,6 @@ class Project < ApplicationRecord Projects::GitGarbageCollectWorker end - def inherited_issuable_templates_enabled? - Feature.enabled?(:inherited_issuable_templates, self, default_enabled: :yaml) - end - def activity_path Gitlab::Routing.url_helpers.activity_project_path(self) end @@ -2618,6 +2617,19 @@ class Project < ApplicationRecord ProjectStatistics.increment_statistic(self, statistic, delta) end + def merge_requests_author_approval + !!read_attribute(:merge_requests_author_approval) + end + + def container_registry_enabled + if Feature.enabled?(:read_container_registry_access_level, self.namespace, default_enabled: :yaml) + project_feature.container_registry_enabled? + else + read_attribute(:container_registry_enabled) + end + end + alias_method :container_registry_enabled?, :container_registry_enabled + private def set_container_registry_access_level @@ -2647,7 +2659,7 @@ class Project < ApplicationRecord end def build_service(name) - Integration.service_name_to_model(name).new(project_id: id) + Integration.integration_name_to_model(name).new(project_id: id) end def services_templates diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index 1fed166e4d0..64e768007ee 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -29,6 +29,15 @@ class ProjectAuthorization < ApplicationRecord EOF end end + + # This method overrides its ActiveRecord's version in order to work correctly + # with composite primary keys and fix the tests for Rails 6.1 + # + # Consider using BulkInsertSafe module instead since we plan to refactor it in + # https://gitlab.com/gitlab-org/gitlab/-/issues/331264 + def self.insert_all(attributes) + super(attributes, unique_by: connection.schema_cache.primary_keys(table_name)) + end end ProjectAuthorization.prepend_mod_with('ProjectAuthorization') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index c0c2ea42d46..b025326c6f8 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -16,6 +16,7 @@ class ProjectCiCdSetting < ApplicationRecord allow_nil: true default_value_for :forward_deployment_enabled, true + default_value_for :job_token_scope_enabled, true def forward_deployment_enabled? super && ::Feature.enabled?(:forward_deployment_enabled, project, default_enabled: true) diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index eb4ad327438..f6e889396c6 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -24,7 +24,11 @@ class ProjectFeature < ApplicationRecord set_available_features(FEATURES) - PRIVATE_FEATURES_MIN_ACCESS_LEVEL = { merge_requests: Gitlab::Access::REPORTER, metrics_dashboard: Gitlab::Access::REPORTER }.freeze + PRIVATE_FEATURES_MIN_ACCESS_LEVEL = { + merge_requests: Gitlab::Access::REPORTER, + metrics_dashboard: Gitlab::Access::REPORTER, + container_registry: Gitlab::Access::REPORTER + }.freeze PRIVATE_FEATURES_MIN_ACCESS_LEVEL_FOR_PRIVATE_PROJECT = { repository: Gitlab::Access::REPORTER }.freeze class << self @@ -92,7 +96,7 @@ class ProjectFeature < ApplicationRecord def set_container_registry_access_level self.container_registry_access_level = - if project&.container_registry_enabled + if project&.read_attribute(:container_registry_enabled) ENABLED else DISABLED diff --git a/app/models/project_feature_usage.rb b/app/models/project_feature_usage.rb index d993db860c3..dba81a6cb60 100644 --- a/app/models/project_feature_usage.rb +++ b/app/models/project_feature_usage.rb @@ -20,14 +20,16 @@ class ProjectFeatureUsage < ApplicationRecord end def log_jira_dvcs_integration_usage(cloud: true) - integration_field = self.class.jira_dvcs_integration_field(cloud: cloud) + ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do + integration_field = self.class.jira_dvcs_integration_field(cloud: cloud) - # The feature usage is used only once later to query the feature usage in a - # long date range. Therefore, we just need to update the timestamp once per - # day - return if persisted? && updated_today?(integration_field) + # The feature usage is used only once later to query the feature usage in a + # long date range. Therefore, we just need to update the timestamp once per + # day + break if persisted? && updated_today?(integration_field) - persist_jira_dvcs_usage(integration_field) + persist_jira_dvcs_usage(integration_field) + end end private diff --git a/app/models/project_repository_storage_move.rb b/app/models/project_repository_storage_move.rb deleted file mode 100644 index e54489ddb88..00000000000 --- a/app/models/project_repository_storage_move.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -# This is a compatibility class to avoid calling a non-existent -# class from sidekiq during deployment. -# -# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. -# we cannot remove this class entirely because there can be jobs -# referencing it. -# -# We can get rid of this class in 14.0 -# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 -class ProjectRepositoryStorageMove < Projects::RepositoryStorageMove -end diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb deleted file mode 100644 index d1c56d2a4d5..00000000000 --- a/app/models/project_services/bugzilla_service.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -class BugzillaService < IssueTrackerService - include ActionView::Helpers::UrlHelper - - validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - - def title - 'Bugzilla' - end - - def description - s_("IssueTracker|Use Bugzilla as this project's issue tracker.") - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bugzilla'), target: '_blank', rel: 'noopener noreferrer' - s_("IssueTracker|Use Bugzilla as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'bugzilla' - end -end diff --git a/app/models/project_services/buildkite_service.rb b/app/models/project_services/buildkite_service.rb deleted file mode 100644 index f2ea5066e37..00000000000 --- a/app/models/project_services/buildkite_service.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -require "addressable/uri" - -class BuildkiteService < CiService - include ReactiveService - - ENDPOINT = "https://buildkite.com" - - prop_accessor :project_url, :token - - validates :project_url, presence: true, public_url: true, if: :activated? - validates :token, presence: true, if: :activated? - - after_save :compose_service_hook, if: :activated? - - def self.supported_events - %w(push merge_request tag_push) - end - - # This is a stub method to work with deprecated API response - # TODO: remove enable_ssl_verification after 14.0 - # https://gitlab.com/gitlab-org/gitlab/-/issues/222808 - def enable_ssl_verification - true - end - - # Since SSL verification will always be enabled for Buildkite, - # we no longer needs to store the boolean. - # This is a stub method to work with deprecated API param. - # TODO: remove enable_ssl_verification after 14.0 - # https://gitlab.com/gitlab-org/gitlab/-/issues/222808 - def enable_ssl_verification=(_value) - self.properties.delete('enable_ssl_verification') # Remove unused key - end - - def webhook_url - "#{buildkite_endpoint('webhook')}/deliver/#{webhook_token}" - end - - def compose_service_hook - hook = service_hook || build_service_hook - hook.url = webhook_url - hook.enable_ssl_verification = true - hook.save - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - service_hook.execute(data) - end - - def commit_status(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } - end - - def commit_status_path(sha) - "#{buildkite_endpoint('gitlab')}/status/#{status_token}.json?commit=#{sha}" - end - - def build_page(sha, ref) - "#{project_url}/builds?commit=#{sha}" - end - - def title - 'Buildkite' - end - - def description - 'Run CI/CD pipelines with Buildkite.' - end - - def self.to_param - 'buildkite' - end - - def fields - [ - { type: 'text', - name: 'token', - title: 'Integration Token', - help: 'This token will be provided when you create a Buildkite pipeline with a GitLab repository', - required: true }, - - { type: 'text', - name: 'project_url', - title: 'Pipeline URL', - placeholder: "#{ENDPOINT}/acme-inc/test-pipeline", - required: true } - ] - end - - def calculate_reactive_cache(sha, ref) - response = Gitlab::HTTP.try_get(commit_status_path(sha), request_options) - - status = - if response&.code == 200 && response['status'] - response['status'] - else - :error - end - - { commit_status: status } - end - - private - - def webhook_token - token_parts.first - end - - def status_token - token_parts.second - end - - def token_parts - if token.present? - token.split(':') - else - [] - end - end - - def buildkite_endpoint(subdomain = nil) - if subdomain.present? - uri = Addressable::URI.parse(ENDPOINT) - new_endpoint = "#{uri.scheme || 'http'}://#{subdomain}.#{uri.host}" - - if uri.port.present? - "#{new_endpoint}:#{uri.port}" - else - new_endpoint - end - else - ENDPOINT - end - end - - def request_options - { verify: false, extra_log_info: { project_id: project_id } } - end -end diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb deleted file mode 100644 index 2f841bf903e..00000000000 --- a/app/models/project_services/chat_notification_service.rb +++ /dev/null @@ -1,252 +0,0 @@ -# frozen_string_literal: true - -# Base class for Chat notifications services -# This class is not meant to be used directly, but only to inherit from. -class ChatNotificationService < Integration - include ChatMessage - include NotificationBranchSelection - - SUPPORTED_EVENTS = %w[ - push issue confidential_issue merge_request note confidential_note - tag_push pipeline wiki_page deployment - ].freeze - - SUPPORTED_EVENTS_FOR_LABEL_FILTER = %w[issue confidential_issue merge_request note confidential_note].freeze - - EVENT_CHANNEL = proc { |event| "#{event}_channel" } - - LABEL_NOTIFICATION_BEHAVIOURS = [ - MATCH_ANY_LABEL = 'match_any', - MATCH_ALL_LABELS = 'match_all' - ].freeze - - default_value_for :category, 'chat' - - prop_accessor :webhook, :username, :channel, :branches_to_be_notified, :labels_to_be_notified, :labels_to_be_notified_behavior - - # Custom serialized properties initialization - prop_accessor(*SUPPORTED_EVENTS.map { |event| EVENT_CHANNEL[event] }) - - boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch - - validates :webhook, presence: true, public_url: true, if: :activated? - validates :labels_to_be_notified_behavior, inclusion: { in: LABEL_NOTIFICATION_BEHAVIOURS }, allow_blank: true - - def initialize_properties - if properties.nil? - self.properties = {} - self.notify_only_broken_pipelines = true - self.branches_to_be_notified = "default" - self.labels_to_be_notified_behavior = MATCH_ANY_LABEL - elsif !self.notify_only_default_branch.nil? - # In older versions, there was only a boolean property named - # `notify_only_default_branch`. Now we have a string property named - # `branches_to_be_notified`. Instead of doing a background migration, we - # opted to set a value for the new property based on the old one, if - # users hasn't specified one already. When users edit the service and - # selects a value for this new property, it will override everything. - - self.branches_to_be_notified ||= notify_only_default_branch? ? "default" : "all" - end - end - - def confidential_issue_channel - properties['confidential_issue_channel'].presence || properties['issue_channel'] - end - - def confidential_note_channel - properties['confidential_note_channel'].presence || properties['note_channel'] - end - - def self.supported_events - SUPPORTED_EVENTS - end - - def fields - default_fields + build_event_channels - end - - def default_fields - [ - { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}", required: true }.freeze, - { type: 'text', name: 'username', placeholder: 'GitLab-integration' }.freeze, - { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'Do not send notifications for successful pipelines.' }.freeze, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }.freeze, - { - type: 'text', - name: 'labels_to_be_notified', - placeholder: '~backend,~frontend', - help: 'Send notifications for issue, merge request, and comment events with the listed labels only. Leave blank to receive notifications for all events.' - }.freeze, - { - type: 'select', - name: 'labels_to_be_notified_behavior', - choices: [ - ['Match any of the labels', MATCH_ANY_LABEL], - ['Match all of the labels', MATCH_ALL_LABELS] - ] - }.freeze - ].freeze - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - return unless notify_label?(data) - - return unless webhook.present? - - object_kind = data[:object_kind] - - data = custom_data(data) - - # WebHook events often have an 'update' event that follows a 'open' or - # 'close' action. Ignore update events for now to prevent duplicate - # messages from arriving. - - message = get_message(object_kind, data) - - return false unless message - - event_type = data[:event_type] || object_kind - - channel_names = get_channel_field(event_type).presence || channel.presence - channels = channel_names&.split(',')&.map(&:strip) - - opts = {} - opts[:channel] = channels if channels.present? - opts[:username] = username if username - - if notify(message, opts) - log_usage(event_type, user_id_from_hook_data(data)) - return true - end - - false - end - - def event_channel_names - supported_events.map { |event| event_channel_name(event) } - end - - def event_field(event) - fields.find { |field| field[:name] == event_channel_name(event) } - end - - def global_fields - fields.reject { |field| field[:name].end_with?('channel') } - end - - def default_channel_placeholder - raise NotImplementedError - end - - private - - def log_usage(_, _) - # Implement in child class - end - - def labels_to_be_notified_list - return [] if labels_to_be_notified.nil? - - labels_to_be_notified.delete('~').split(',').map(&:strip) - end - - def notify_label?(data) - return true unless SUPPORTED_EVENTS_FOR_LABEL_FILTER.include?(data[:object_kind]) && labels_to_be_notified.present? - - labels = data.dig(:issue, :labels) || data.dig(:merge_request, :labels) - - return false if labels.nil? - - matching_labels = labels_to_be_notified_list & labels.pluck(:title) - - if labels_to_be_notified_behavior == MATCH_ALL_LABELS - labels_to_be_notified_list.difference(matching_labels).empty? - else - matching_labels.any? - end - end - - def user_id_from_hook_data(data) - data.dig(:user, :id) || data[:user_id] - end - - # every notifier must implement this independently - def notify(message, opts) - raise NotImplementedError - end - - def custom_data(data) - data.merge(project_url: project_url, project_name: project_name) - end - - def get_message(object_kind, data) - case object_kind - when "push", "tag_push" - Integrations::ChatMessage::PushMessage.new(data) if notify_for_ref?(data) - when "issue" - Integrations::ChatMessage::IssueMessage.new(data) unless update?(data) - when "merge_request" - Integrations::ChatMessage::MergeMessage.new(data) unless update?(data) - when "note" - Integrations::ChatMessage::NoteMessage.new(data) - when "pipeline" - Integrations::ChatMessage::PipelineMessage.new(data) if should_pipeline_be_notified?(data) - when "wiki_page" - Integrations::ChatMessage::WikiPageMessage.new(data) - when "deployment" - Integrations::ChatMessage::DeploymentMessage.new(data) - end - end - - def get_channel_field(event) - field_name = event_channel_name(event) - self.public_send(field_name) # rubocop:disable GitlabSecurity/PublicSend - end - - def build_event_channels - supported_events.reduce([]) do |channels, event| - channels << { type: 'text', name: event_channel_name(event), placeholder: default_channel_placeholder } - end - end - - def event_channel_name(event) - EVENT_CHANNEL[event] - end - - def project_name - project.full_name - end - - def project_url - project.web_url - end - - def update?(data) - data[:object_attributes][:action] == 'update' - end - - def should_pipeline_be_notified?(data) - notify_for_ref?(data) && notify_for_pipeline?(data) - end - - def notify_for_ref?(data) - return true if data[:object_kind] == 'tag_push' - return true if data.dig(:object_attributes, :tag) - - notify_for_branch?(data) - end - - def notify_for_pipeline?(data) - case data[:object_attributes][:status] - when 'success' - !notify_only_broken_pipelines? - when 'failed' - true - else - false - end - end -end diff --git a/app/models/project_services/ci_service.rb b/app/models/project_services/ci_service.rb deleted file mode 100644 index 0733da761d5..00000000000 --- a/app/models/project_services/ci_service.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -# Base class for CI services -# List methods you need to implement to get your CI service -# working with GitLab merge requests -class CiService < Integration - default_value_for :category, 'ci' - - def valid_token?(token) - self.respond_to?(:token) && self.token.present? && ActiveSupport::SecurityUtils.secure_compare(token, self.token) - end - - def self.supported_events - %w(push) - end - - # Return complete url to build page - # - # Ex. - # http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c - # - def build_page(sha, ref) - # implement inside child - end - - # Return string with build status or :error symbol - # - # Allowed states: 'success', 'failed', 'running', 'pending', 'skipped' - # - # - # Ex. - # @service.commit_status('13be4ac', 'master') - # # => 'success' - # - # @service.commit_status('2abe4ac', 'dev') - # # => 'running' - # - # - def commit_status(sha, ref) - # implement inside child - end -end diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb deleted file mode 100644 index 6f99d104904..00000000000 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class CustomIssueTrackerService < IssueTrackerService - include ActionView::Helpers::UrlHelper - validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - - def title - s_('IssueTracker|Custom issue tracker') - end - - def description - s_("IssueTracker|Use a custom issue tracker as this project's issue tracker.") - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/custom_issue_tracker'), target: '_blank', rel: 'noopener noreferrer' - s_('IssueTracker|Use a custom issue tracker that is not in the integration list. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'custom_issue_tracker' - end -end diff --git a/app/models/project_services/data_fields.rb b/app/models/project_services/data_fields.rb deleted file mode 100644 index ca4dc0375fb..00000000000 --- a/app/models/project_services/data_fields.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module DataFields - extend ActiveSupport::Concern - - class_methods do - # Provide convenient accessor methods for data fields. - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - def data_field(*args) - args.each do |arg| - self.class_eval <<-RUBY, __FILE__, __LINE__ + 1 - unless method_defined?(arg) - def #{arg} - data_fields.send('#{arg}') || (properties && properties['#{arg}']) - end - end - - def #{arg}=(value) - @old_data_fields ||= {} - @old_data_fields['#{arg}'] ||= #{arg} # set only on the first assignment, IOW we remember the original value only - data_fields.send('#{arg}=', value) - end - - def #{arg}_touched? - @old_data_fields ||= {} - @old_data_fields.has_key?('#{arg}') - end - - def #{arg}_changed? - #{arg}_touched? && @old_data_fields['#{arg}'] != #{arg} - end - - def #{arg}_was - return unless #{arg}_touched? - return if data_fields.persisted? # arg_was does not work for attr_encrypted - - legacy_properties_data['#{arg}'] - end - RUBY - end - end - end - - included do - has_one :issue_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id - has_one :jira_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id - has_one :open_project_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id - - def data_fields - raise NotImplementedError - end - - def data_fields_present? - data_fields.present? - rescue NotImplementedError - false - end - end -end diff --git a/app/models/project_services/discord_service.rb b/app/models/project_services/discord_service.rb deleted file mode 100644 index d7adf63fde4..00000000000 --- a/app/models/project_services/discord_service.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -require "discordrb/webhooks" - -class DiscordService < ChatNotificationService - include ActionView::Helpers::UrlHelper - - ATTACHMENT_REGEX = /: (?<entry>.*?)\n - (?<name>.*)\n*/.freeze - - def title - s_("DiscordService|Discord Notifications") - end - - def description - s_("DiscordService|Send notifications about project events to a Discord channel.") - end - - def self.to_param - "discord" - end - - def help - docs_link = link_to _('How do I set up this service?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/discord_notifications'), target: '_blank', rel: 'noopener noreferrer' - s_('Send notifications about project events to a Discord channel. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def event_field(event) - # No-op. - end - - def default_channel_placeholder - # No-op. - end - - def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push pipeline wiki_page] - end - - def default_fields - [ - { type: "text", name: "webhook", placeholder: "https://discordapp.com/api/webhooks/…", help: "URL to the webhook for the Discord channel." }, - { type: "checkbox", name: "notify_only_broken_pipelines" }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } - ] - end - - private - - def notify(message, opts) - client = Discordrb::Webhooks::Client.new(url: webhook) - - client.execute do |builder| - builder.add_embed do |embed| - embed.author = Discordrb::Webhooks::EmbedAuthor.new(name: message.user_name, icon_url: message.user_avatar) - embed.description = (message.pretext + "\n" + Array.wrap(message.attachments).join("\n")).gsub(ATTACHMENT_REGEX, " \\k<entry> - \\k<name>\n") - end - end - rescue RestClient::Exception => error - log_error(error.message) - false - end - - def custom_data(data) - super(data).merge(markdown: true) - end -end diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb deleted file mode 100644 index ab1ba768a8f..00000000000 --- a/app/models/project_services/drone_ci_service.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -class DroneCiService < CiService - include ReactiveService - include ServicePushDataValidations - - prop_accessor :drone_url, :token - boolean_accessor :enable_ssl_verification - - validates :drone_url, presence: true, public_url: true, if: :activated? - validates :token, presence: true, if: :activated? - - after_save :compose_service_hook, if: :activated? - - def compose_service_hook - hook = service_hook || build_service_hook - # If using a service template, project may not be available - hook.url = [drone_url, "/api/hook", "?owner=#{project.namespace.full_path}", "&name=#{project.path}", "&access_token=#{token}"].join if project - hook.enable_ssl_verification = !!enable_ssl_verification - hook.save - end - - def execute(data) - case data[:object_kind] - when 'push' - service_hook.execute(data) if push_valid?(data) - when 'merge_request' - service_hook.execute(data) if merge_request_valid?(data) - when 'tag_push' - service_hook.execute(data) if tag_push_valid?(data) - end - end - - def allow_target_ci? - true - end - - def self.supported_events - %w(push merge_request tag_push) - end - - def commit_status_path(sha, ref) - Gitlab::Utils.append_path( - drone_url, - "gitlab/#{project.full_path}/commits/#{sha}?branch=#{Addressable::URI.encode_component(ref.to_s)}&access_token=#{token}") - end - - def commit_status(sha, ref) - with_reactive_cache(sha, ref) { |cached| cached[:commit_status] } - end - - def calculate_reactive_cache(sha, ref) - response = Gitlab::HTTP.try_get(commit_status_path(sha, ref), - verify: enable_ssl_verification, - extra_log_info: { project_id: project_id }) - - status = - if response && response.code == 200 && response['status'] - case response['status'] - when 'killed' - :canceled - when 'failure', 'error' - # Because drone return error if some test env failed - :failed - else - response["status"] - end - else - :error - end - - { commit_status: status } - end - - def build_page(sha, ref) - Gitlab::Utils.append_path( - drone_url, - "gitlab/#{project.full_path}/redirect/commits/#{sha}?branch=#{Addressable::URI.encode_component(ref.to_s)}") - end - - def title - 'Drone' - end - - def description - s_('ProjectService|Run CI/CD pipelines with Drone.') - end - - def self.to_param - 'drone_ci' - end - - def help - s_('ProjectService|Run CI/CD pipelines with Drone.') - end - - def fields - [ - { type: 'text', name: 'token', help: s_('ProjectService|Token for the Drone project.'), required: true }, - { type: 'text', name: 'drone_url', title: s_('ProjectService|Drone server URL'), placeholder: 'http://drone.example.com', required: true }, - { type: 'checkbox', name: 'enable_ssl_verification', title: "Enable SSL verification" } - ] - end -end diff --git a/app/models/project_services/ewm_service.rb b/app/models/project_services/ewm_service.rb deleted file mode 100644 index 90fcbb10d2b..00000000000 --- a/app/models/project_services/ewm_service.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -class EwmService < IssueTrackerService - include ActionView::Helpers::UrlHelper - - validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - - def self.reference_pattern(only_long: true) - @reference_pattern ||= %r{(?<issue>\b(bug|task|work item|workitem|rtcwi|defect)\b\s+\d+)}i - end - - def title - 'EWM' - end - - def description - s_("IssueTracker|Use IBM Engineering Workflow Management as this project's issue tracker.") - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/ewm'), target: '_blank', rel: 'noopener noreferrer' - s_("IssueTracker|Use IBM Engineering Workflow Management as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'ewm' - end - - def can_test? - false - end - - def issue_url(iid) - issues_url.gsub(':id', iid.to_s.split(' ')[-1]) - end -end diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb deleted file mode 100644 index f49b008533d..00000000000 --- a/app/models/project_services/external_wiki_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -class ExternalWikiService < Integration - include ActionView::Helpers::UrlHelper - - prop_accessor :external_wiki_url - validates :external_wiki_url, presence: true, public_url: true, if: :activated? - - def title - s_('ExternalWikiService|External wiki') - end - - def description - s_('ExternalWikiService|Link to an external wiki from the sidebar.') - end - - def self.to_param - 'external_wiki' - end - - def fields - [ - { - type: 'text', - name: 'external_wiki_url', - title: s_('ExternalWikiService|External wiki URL'), - placeholder: s_('ExternalWikiService|https://example.com/xxx/wiki/...'), - help: 'Enter the URL to the external wiki.', - required: true - } - ] - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/wiki/index', anchor: 'link-an-external-wiki'), target: '_blank', rel: 'noopener noreferrer' - - s_('Link an external wiki from the project\'s sidebar. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def execute(_data) - response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true) - response.body if response.code == 200 - rescue StandardError - nil - end - - def self.supported_events - %w() - end -end diff --git a/app/models/project_services/flowdock_service.rb b/app/models/project_services/flowdock_service.rb deleted file mode 100644 index 7aae5af7454..00000000000 --- a/app/models/project_services/flowdock_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -class FlowdockService < Integration - include ActionView::Helpers::UrlHelper - - prop_accessor :token - validates :token, presence: true, if: :activated? - - def title - 'Flowdock' - end - - def description - s_('FlowdockService|Send event notifications from GitLab to Flowdock flows.') - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'flowdock'), target: '_blank', rel: 'noopener noreferrer' - s_('FlowdockService|Send event notifications from GitLab to Flowdock flows. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'flowdock' - end - - def fields - [ - { type: 'text', name: 'token', placeholder: s_('FlowdockService|1b609b52537...'), required: true, help: 'Enter your Flowdock token.' } - ] - end - - def self.supported_events - %w(push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - Flowdock::Git.post( - data[:ref], - data[:before], - data[:after], - token: token, - repo: project.repository, - repo_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}", - commit_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/-/commit/%s", - diff_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/compare/%s...%s" - ) - end -end diff --git a/app/models/project_services/hangouts_chat_service.rb b/app/models/project_services/hangouts_chat_service.rb deleted file mode 100644 index 6e7708a169f..00000000000 --- a/app/models/project_services/hangouts_chat_service.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -require 'hangouts_chat' - -class HangoutsChatService < ChatNotificationService - include ActionView::Helpers::UrlHelper - - def title - 'Google Chat' - end - - def description - 'Send notifications from GitLab to a room in Google Chat.' - end - - def self.to_param - 'hangouts_chat' - end - - def help - docs_link = link_to _('How do I set up a Google Chat webhook?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/hangouts_chat'), target: '_blank', rel: 'noopener noreferrer' - s_('Before enabling this integration, create a webhook for the room in Google Chat where you want to receive notifications from this project. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def event_field(event) - end - - def default_channel_placeholder - end - - def webhook_placeholder - 'https://chat.googleapis.com/v1/spaces…' - end - - def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push - pipeline wiki_page] - end - - def default_fields - [ - { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } - ] - end - - private - - def notify(message, opts) - simple_text = parse_simple_text_message(message) - HangoutsChat::Sender.new(webhook).simple(simple_text) - end - - def parse_simple_text_message(message) - header = message.pretext - return header if message.attachments.empty? - - attachment = message.attachments.first - title = format_attachment_title(attachment) - body = attachment[:text] - - [header, title, body].compact.join("\n") - end - - def format_attachment_title(attachment) - return attachment[:title] unless attachment[:title_link] - - "<#{attachment[:title_link]}|#{attachment[:title]}>" - end -end diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb deleted file mode 100644 index 71d8e7bfac4..00000000000 --- a/app/models/project_services/hipchat_service.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -# This service is scheduled for removal. All records must -# be deleted before the class can be removed. -# https://gitlab.com/gitlab-org/gitlab/-/issues/27954 -class HipchatService < Integration - before_save :prevent_save - - def self.to_param - 'hipchat' - end - - def self.supported_events - [] - end - - def execute(data) - # We removed the hipchat gem due to https://gitlab.com/gitlab-org/gitlab/-/issues/325851#note_537143149 - # HipChat is unusable anyway, so do nothing in this method - end - - private - - def prevent_save - errors.add(:base, _('HipChat endpoint is deprecated and should not be created or modified.')) - - # Stops execution of callbacks and database operation while - # preserving expectations of #save (will not raise) & #save! (raises) - # https://guides.rubyonrails.org/active_record_callbacks.html#halting-execution - throw :abort # rubocop:disable Cop/BanCatchThrow - end -end diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb deleted file mode 100644 index 5cca620c659..00000000000 --- a/app/models/project_services/irker_service.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -class IrkerService < Integration - prop_accessor :server_host, :server_port, :default_irc_uri - prop_accessor :recipients, :channels - boolean_accessor :colorize_messages - validates :recipients, presence: true, if: :validate_recipients? - - before_validation :get_channels - - def title - 'Irker (IRC gateway)' - end - - def description - 'Send IRC messages.' - end - - def self.to_param - 'irker' - end - - def self.supported_events - %w(push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - IrkerWorker.perform_async(project_id, channels, - colorize_messages, data, settings) - end - - def settings - { - server_host: server_host.presence || 'localhost', - server_port: server_port.presence || 6659 - } - end - - def fields - [ - { type: 'text', name: 'server_host', placeholder: 'localhost', - help: 'Irker daemon hostname (defaults to localhost)' }, - { type: 'text', name: 'server_port', placeholder: 6659, - help: 'Irker daemon port (defaults to 6659)' }, - { type: 'text', name: 'default_irc_uri', title: 'Default IRC URI', - help: 'A default IRC URI to prepend before each recipient (optional)', - placeholder: 'irc://irc.network.net:6697/' }, - { type: 'textarea', name: 'recipients', - placeholder: 'Recipients/channels separated by whitespaces', required: true, - help: 'Recipients have to be specified with a full URI: '\ - 'irc[s]://irc.network.net[:port]/#channel. Special cases: if '\ - 'you want the channel to be a nickname instead, append ",isnick" to ' \ - 'the channel name; if the channel is protected by a secret password, ' \ - ' append "?key=secretpassword" to the URI (Note that due to a bug, if you ' \ - ' want to use a password, you have to omit the "#" on the channel). If you ' \ - ' specify a default IRC URI to prepend before each recipient, you can just ' \ - ' give a channel name.' }, - { type: 'checkbox', name: 'colorize_messages' } - ] - end - - def help - ' NOTE: Irker does NOT have built-in authentication, which makes it' \ - ' vulnerable to spamming IRC channels if it is hosted outside of a ' \ - ' firewall. Please make sure you run the daemon within a secured network ' \ - ' to prevent abuse. For more details, read: http://www.catb.org/~esr/irker/security.html.' - end - - private - - def get_channels - return true unless activated? - return true if recipients.nil? || recipients.empty? - - map_recipients - - errors.add(:recipients, 'are all invalid') if channels.empty? - true - end - - def map_recipients - self.channels = recipients.split(/\s+/).map do |recipient| - format_channel(recipient) - end - channels.reject!(&:nil?) - end - - def format_channel(recipient) - uri = nil - - # Try to parse the chan as a full URI - begin - uri = consider_uri(URI.parse(recipient)) - rescue URI::InvalidURIError - end - - unless uri.present? && default_irc_uri.nil? - begin - new_recipient = URI.join(default_irc_uri, '/', recipient).to_s - uri = consider_uri(URI.parse(new_recipient)) - rescue StandardError - log_error("Unable to create a valid URL", default_irc_uri: default_irc_uri, recipient: recipient) - end - end - - uri - end - - def consider_uri(uri) - return if uri.scheme.nil? - - # Authorize both irc://domain.com/#chan and irc://domain.com/chan - if uri.is_a?(URI) && uri.scheme[/^ircs?\z/] && !uri.path.nil? - uri.to_s - end - end -end diff --git a/app/models/project_services/issue_tracker_data.rb b/app/models/project_services/issue_tracker_data.rb deleted file mode 100644 index 414f2c1da4d..00000000000 --- a/app/models/project_services/issue_tracker_data.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class IssueTrackerData < ApplicationRecord - include Services::DataFields - - attr_encrypted :project_url, encryption_options - attr_encrypted :issues_url, encryption_options - attr_encrypted :new_issue_url, encryption_options -end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb deleted file mode 100644 index 099e3c336dd..00000000000 --- a/app/models/project_services/issue_tracker_service.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: true - -class IssueTrackerService < Integration - validate :one_issue_tracker, if: :activated?, on: :manual_change - - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :project_url, :issues_url, :new_issue_url - - default_value_for :category, 'issue_tracker' - - before_validation :handle_properties - before_validation :set_default_data, on: :create - - # Pattern used to extract links from comments - # Override this method on services that uses different patterns - # This pattern does not support cross-project references - # The other code assumes that this pattern is a superset of all - # overridden patterns. See ReferenceRegexes.external_pattern - def self.reference_pattern(only_long: false) - if only_long - /(\b[A-Z][A-Z0-9_]*-)#{Gitlab::Regex.issue}/ - else - /(\b[A-Z][A-Z0-9_]*-|#{Issue.reference_prefix})#{Gitlab::Regex.issue}/ - end - end - - def handle_properties - # this has been moved from initialize_properties and should be improved - # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - return unless properties - - @legacy_properties_data = properties.dup - data_values = properties.slice!('title', 'description') - data_values.reject! { |key| data_fields.changed.include?(key) } - data_values.slice!(*data_fields.attributes.keys) - data_fields.assign_attributes(data_values) if data_values.present? - - self.properties = {} - end - - def legacy_properties_data - @legacy_properties_data ||= {} - end - - def supports_data_fields? - true - end - - def data_fields - issue_tracker_data || self.build_issue_tracker_data - end - - def default? - default - end - - def issue_url(iid) - issues_url.gsub(':id', iid.to_s) - end - - def issue_tracker_path - project_url - end - - def new_issue_path - new_issue_url - end - - def issue_path(iid) - issue_url(iid) - end - - def fields - [ - { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in the external issue tracker.'), required: true }, - { type: 'text', name: 'issues_url', title: s_('IssueTracker|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true }, - { type: 'text', name: 'new_issue_url', title: s_('IssueTracker|New issue URL'), help: s_('IssueTracker|The URL to create an issue in the external issue tracker.'), required: true } - ] - end - - def initialize_properties - {} - end - - # Initialize with default properties values - def set_default_data - return unless issues_tracker.present? - - # we don't want to override if we have set something - return if project_url || issues_url || new_issue_url - - data_fields.project_url = issues_tracker['project_url'] - data_fields.issues_url = issues_tracker['issues_url'] - data_fields.new_issue_url = issues_tracker['new_issue_url'] - end - - def self.supported_events - %w(push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - message = "#{self.type} was unable to reach #{self.project_url}. Check the url and try again." - result = false - - begin - response = Gitlab::HTTP.head(self.project_url, verify: true) - - if response - message = "#{self.type} received response #{response.code} when attempting to connect to #{self.project_url}" - result = true - end - rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => error - message = "#{self.type} had an error when trying to connect to #{self.project_url}: #{error.message}" - end - log_info(message) - result - end - - def support_close_issue? - false - end - - def support_cross_reference? - false - end - - private - - def enabled_in_gitlab_config - Gitlab.config.issues_tracker && - Gitlab.config.issues_tracker.values.any? && - issues_tracker - end - - def issues_tracker - Gitlab.config.issues_tracker[to_param] - end - - def one_issue_tracker - return if template? || instance? - return if project.blank? - - if project.integrations.external_issue_trackers.where.not(id: id).any? - errors.add(:base, _('Another issue tracker is already in use. Only one issue tracker service can be active at a time')) - end - end -end - -IssueTrackerService.prepend_mod_with('IssueTrackerService') diff --git a/app/models/project_services/jenkins_service.rb b/app/models/project_services/jenkins_service.rb deleted file mode 100644 index 990a35cd617..00000000000 --- a/app/models/project_services/jenkins_service.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -class JenkinsService < CiService - include ActionView::Helpers::UrlHelper - - prop_accessor :jenkins_url, :project_name, :username, :password - - before_update :reset_password - - validates :jenkins_url, presence: true, addressable_url: true, if: :activated? - validates :project_name, presence: true, if: :activated? - validates :username, presence: true, if: ->(service) { service.activated? && service.password_touched? && service.password.present? } - - default_value_for :push_events, true - default_value_for :merge_requests_events, false - default_value_for :tag_push_events, false - - after_save :compose_service_hook, if: :activated? - - def reset_password - # don't reset the password if a new one is provided - if (jenkins_url_changed? || username.blank?) && !password_touched? - self.password = nil - end - end - - def compose_service_hook - hook = service_hook || build_service_hook - hook.url = hook_url - hook.save - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - service_hook.execute(data, "#{data[:object_kind]}_hook") - end - - def test(data) - begin - result = execute(data) - return { success: false, result: result[:message] } if result[:http_status] != 200 - rescue StandardError => error - return { success: false, result: error } - end - - { success: true, result: result[:message] } - end - - def hook_url - url = URI.parse(jenkins_url) - url.path = File.join(url.path || '/', "project/#{project_name}") - url.user = ERB::Util.url_encode(username) unless username.blank? - url.password = ERB::Util.url_encode(password) unless password.blank? - url.to_s - end - - def self.supported_events - %w(push merge_request tag_push) - end - - def title - 'Jenkins' - end - - def description - s_('Run CI/CD pipelines with Jenkins.') - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('integration/jenkins'), target: '_blank', rel: 'noopener noreferrer' - s_('Run CI/CD pipelines with Jenkins when you push to a repository, or when a merge request is created, updated, or merged. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'jenkins' - end - - def fields - [ - { - type: 'text', - name: 'jenkins_url', - title: s_('ProjectService|Jenkins server URL'), - required: true, - placeholder: 'http://jenkins.example.com', - help: s_('The URL of the Jenkins server.') - }, - { - type: 'text', - name: 'project_name', - required: true, - placeholder: 'my_project_name', - help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') - }, - { - type: 'text', - name: 'username', - required: true, - help: s_('The username for the Jenkins server.') - }, - { - type: 'password', - name: 'password', - help: s_('The password for the Jenkins server.'), - non_empty_password_title: s_('ProjectService|Enter new password.'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') - } - ] - end -end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb deleted file mode 100644 index 5cd6e79eb1d..00000000000 --- a/app/models/project_services/jira_service.rb +++ /dev/null @@ -1,541 +0,0 @@ -# frozen_string_literal: true - -# Accessible as Project#external_issue_tracker -class JiraService < IssueTrackerService - extend ::Gitlab::Utils::Override - include Gitlab::Routing - include ApplicationHelper - include ActionView::Helpers::AssetUrlHelper - include Gitlab::Utils::StrongMemoize - - PROJECTS_PER_PAGE = 50 - - # TODO: use jira_service.deployment_type enum when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37003 is merged - DEPLOYMENT_TYPES = { - server: 'SERVER', - cloud: 'CLOUD' - }.freeze - - validates :url, public_url: true, presence: true, if: :activated? - validates :api_url, public_url: true, allow_blank: true - validates :username, presence: true, if: :activated? - validates :password, presence: true, if: :activated? - - validates :jira_issue_transition_id, - format: { with: Gitlab::Regex.jira_transition_id_regex, message: s_("JiraService|transition ids can have only numbers which can be split with , or ;") }, - allow_blank: true - - # Jira Cloud version is deprecating authentication via username and password. - # We should use username/password for Jira Server and email/api_token for Jira Cloud, - # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. - - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_automatic, :jira_issue_transition_id, :project_key, :issues_enabled, - :vulnerabilities_enabled, :vulnerabilities_issuetype - - before_update :reset_password - after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? - - enum comment_detail: { - standard: 1, - all_details: 2 - } - - alias_method :project_url, :url - - # When these are false GitLab does not create cross reference - # comments on Jira except when an issue gets transitioned. - def self.supported_events - %w(commit merge_request) - end - - def self.supported_event_actions - %w(comment) - end - - # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 - def self.reference_pattern(only_long: true) - @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ - end - - def initialize_properties - {} - end - - def data_fields - jira_tracker_data || self.build_jira_tracker_data - end - - def reset_password - data_fields.password = nil if reset_password? - end - - def set_default_data - return unless issues_tracker.present? - - return if url - - data_fields.url ||= issues_tracker['url'] - data_fields.api_url ||= issues_tracker['api_url'] - end - - def options - url = URI.parse(client_url) - - { - username: username&.strip, - password: password, - site: URI.join(url, '/').to_s, # Intended to find the root - context_path: url.path, - auth_type: :basic, - read_timeout: 120, - use_cookies: true, - additional_cookies: ['OBBasicAuth=fromDialog'], - use_ssl: url.scheme == 'https' - } - end - - def client - @client ||= begin - JIRA::Client.new(options).tap do |client| - # Replaces JIRA default http client with our implementation - client.request_client = Gitlab::Jira::HttpClient.new(client.options) - end - end - end - - def help - jira_doc_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_url('integration/jira/index.html') } - s_("JiraService|You need to configure Jira before enabling this integration. For more details, read the %{jira_doc_link_start}Jira integration documentation%{link_end}.") % { jira_doc_link_start: jira_doc_link_start, link_end: '</a>'.html_safe } - end - - def title - 'Jira' - end - - def description - s_("JiraService|Use Jira as this project's issue tracker.") - end - - def self.to_param - 'jira' - end - - def fields - [ - { - type: 'text', - name: 'url', - title: s_('JiraService|Web URL'), - placeholder: 'https://jira.example.com', - help: s_('JiraService|Base URL of the Jira instance.'), - required: true - }, - { - type: 'text', - name: 'api_url', - title: s_('JiraService|Jira API URL'), - help: s_('JiraService|If different from Web URL.') - }, - { - type: 'text', - name: 'username', - title: s_('JiraService|Username or Email'), - help: s_('JiraService|Use a username for server version and an email for cloud version.'), - required: true - }, - { - type: 'password', - name: 'password', - title: s_('JiraService|Password or API token'), - non_empty_password_title: s_('JiraService|Enter new password or API token'), - non_empty_password_help: s_('JiraService|Leave blank to use your current password or API token.'), - help: s_('JiraService|Use a password for server version and an API token for cloud version.'), - required: true - } - ] - end - - def issues_url - "#{url}/browse/:id" - end - - def new_issue_url - "#{url}/secure/CreateIssue!default.jspa" - end - - alias_method :original_url, :url - def url - original_url&.delete_suffix('/') - end - - alias_method :original_api_url, :api_url - def api_url - original_api_url&.delete_suffix('/') - end - - def execute(push) - # This method is a no-op, because currently JiraService does not - # support any events. - end - - def find_issue(issue_key, rendered_fields: false, transitions: false) - expands = [] - expands << 'renderedFields' if rendered_fields - expands << 'transitions' if transitions - options = { expand: expands.join(',') } if expands.any? - - jira_request { client.Issue.find(issue_key, options || {}) } - end - - def close_issue(entity, external_issue, current_user) - issue = find_issue(external_issue.iid, transitions: jira_issue_transition_automatic) - - return if issue.nil? || has_resolution?(issue) || !issue_transition_enabled? - - commit_id = case entity - when Commit then entity.id - when MergeRequest then entity.diff_head_sha - end - - commit_url = build_entity_url(:commit, commit_id) - - # Depending on the Jira project's workflow, a comment during transition - # may or may not be allowed. Refresh the issue after transition and check - # if it is closed, so we don't have one comment for every commit. - issue = find_issue(issue.key) if transition_issue(issue) - add_issue_solved_comment(issue, commit_id, commit_url) if has_resolution?(issue) - log_usage(:close_issue, current_user) - end - - def create_cross_reference_note(mentioned, noteable, author) - unless can_cross_reference?(noteable) - return s_("JiraService|Events for %{noteable_model_name} are disabled.") % { noteable_model_name: noteable.model_name.plural.humanize(capitalize: false) } - end - - jira_issue = find_issue(mentioned.id) - - return unless jira_issue.present? - - noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id - noteable_type = noteable_name(noteable) - entity_url = build_entity_url(noteable_type, noteable_id) - entity_meta = build_entity_meta(noteable) - - data = { - user: { - name: author.name, - url: resource_url(user_path(author)) - }, - project: { - name: project.full_path, - url: resource_url(project_path(project)) - }, - entity: { - id: entity_meta[:id], - name: noteable_type.humanize.downcase, - url: entity_url, - title: noteable.title, - description: entity_meta[:description], - branch: entity_meta[:branch] - } - } - - add_comment(data, jira_issue).tap { log_usage(:cross_reference, author) } - end - - def valid_connection? - test(nil)[:success] - end - - def test(_) - result = server_info - success = result.present? - result = @error&.message unless success - - { success: success, result: result } - end - - override :support_close_issue? - def support_close_issue? - true - end - - override :support_cross_reference? - def support_cross_reference? - true - end - - def issue_transition_enabled? - jira_issue_transition_automatic || jira_issue_transition_id.present? - end - - private - - def server_info - strong_memoize(:server_info) do - client_url.present? ? jira_request { client.ServerInfo.all.attrs } : nil - end - end - - def can_cross_reference?(noteable) - case noteable - when Commit then commit_events - when MergeRequest then merge_requests_events - else true - end - end - - # jira_issue_transition_id can have multiple values split by , or ; - # the issue is transitioned at the order given by the user - # if any transition fails it will log the error message and stop the transition sequence - def transition_issue(issue) - return transition_issue_to_done(issue) if jira_issue_transition_automatic - - jira_issue_transition_id.scan(Gitlab::Regex.jira_transition_id_regex).all? do |transition_id| - transition_issue_to_id(issue, transition_id) - end - end - - def transition_issue_to_id(issue, transition_id) - issue.transitions.build.save!( - transition: { id: transition_id } - ) - - true - rescue StandardError => error - log_error( - "Issue transition failed", - error: { - exception_class: error.class.name, - exception_message: error.message, - exception_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(error.backtrace) - }, - client_url: client_url - ) - - false - end - - def transition_issue_to_done(issue) - transitions = issue.transitions rescue [] - - transition = transitions.find do |transition| - status = transition&.to&.statusCategory - status && status['key'] == 'done' - end - - return false unless transition - - transition_issue_to_id(issue, transition.id) - end - - def log_usage(action, user) - key = "i_ecosystem_jira_service_#{action}" - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user.id) - end - - def add_issue_solved_comment(issue, commit_id, commit_url) - link_title = "Solved by commit #{commit_id}." - comment = "Issue solved with [#{commit_id}|#{commit_url}]." - link_props = build_remote_link_props(url: commit_url, title: link_title, resolved: true) - send_message(issue, comment, link_props) - end - - def add_comment(data, issue) - entity_name = data[:entity][:name] - entity_url = data[:entity][:url] - entity_title = data[:entity][:title] - - message = comment_message(data) - link_title = "#{entity_name.capitalize} - #{entity_title}" - link_props = build_remote_link_props(url: entity_url, title: link_title) - - unless comment_exists?(issue, message) - send_message(issue, message, link_props) - end - end - - def comment_message(data) - user_link = build_jira_link(data[:user][:name], data[:user][:url]) - - entity = data[:entity] - entity_ref = all_details? ? "#{entity[:name]} #{entity[:id]}" : "a #{entity[:name]}" - entity_link = build_jira_link(entity_ref, entity[:url]) - - project_link = build_jira_link(project.full_name, Gitlab::Routing.url_helpers.project_url(project)) - branch = - if entity[:branch].present? - s_('JiraService| on branch %{branch_link}') % { - branch_link: build_jira_link(entity[:branch], project_tree_url(project, entity[:branch])) - } - end - - entity_message = entity[:description].presence if all_details? - entity_message ||= entity[:title].chomp - - s_('JiraService|%{user_link} mentioned this issue in %{entity_link} of %{project_link}%{branch}:{quote}%{entity_message}{quote}') % { - user_link: user_link, - entity_link: entity_link, - project_link: project_link, - branch: branch, - entity_message: entity_message - } - end - - def build_jira_link(title, url) - "[#{title}|#{url}]" - end - - def has_resolution?(issue) - issue.respond_to?(:resolution) && issue.resolution.present? - end - - def comment_exists?(issue, message) - comments = jira_request { issue.comments } - - comments.present? && comments.any? { |comment| comment.body.include?(message) } - end - - def send_message(issue, message, remote_link_props) - return unless client_url.present? - - jira_request do - remote_link = find_remote_link(issue, remote_link_props[:object][:url]) - - create_issue_comment(issue, message) unless remote_link - remote_link ||= issue.remotelink.build - remote_link.save!(remote_link_props) - - log_info("Successfully posted", client_url: client_url) - "SUCCESS: Successfully posted to #{client_url}." - end - end - - def create_issue_comment(issue, message) - return unless comment_on_event_enabled - - issue.comments.build.save!(body: message) - end - - def find_remote_link(issue, url) - links = jira_request { issue.remotelink.all } - return unless links - - links.find { |link| link.object["url"] == url } - end - - def build_remote_link_props(url:, title:, resolved: false) - status = { - resolved: resolved - } - - { - GlobalID: 'GitLab', - relationship: 'mentioned on', - object: { - url: url, - title: title, - status: status, - icon: { - title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.base_url) - } - } - } - end - - def resource_url(resource) - "#{Settings.gitlab.base_url.chomp("/")}#{resource}" - end - - def build_entity_url(noteable_type, entity_id) - polymorphic_url( - [ - self.project, - noteable_type.to_sym - ], - id: entity_id, - host: Settings.gitlab.base_url - ) - end - - def build_entity_meta(noteable) - if noteable.is_a?(Commit) - { - id: noteable.short_id, - description: noteable.safe_message, - branch: noteable.ref_names(project.repository).first - } - elsif noteable.is_a?(MergeRequest) - { - id: noteable.to_reference, - branch: noteable.source_branch - } - else - {} - end - end - - def noteable_name(noteable) - name = noteable.model_name.singular - - # ProjectSnippet inherits from Snippet class so it causes - # routing error building the URL. - name == "project_snippet" ? "snippet" : name - end - - # Handle errors when doing Jira API calls - def jira_request - yield - rescue StandardError => error - @error = error - log_error("Error sending message", client_url: client_url, error: @error.message) - nil - end - - def client_url - api_url.presence || url - end - - def reset_password? - # don't reset the password if a new one is provided - return false if password_touched? - return true if api_url_changed? - return false if api_url.present? - - url_changed? - end - - def update_deployment_type? - (api_url_changed? || url_changed? || username_changed? || password_changed?) && - can_test? - end - - def update_deployment_type - clear_memoization(:server_info) # ensure we run the request when we try to update deployment type - results = server_info - return data_fields.deployment_unknown! unless results.present? - - case results['deploymentType'] - when 'Server' - data_fields.deployment_server! - when 'Cloud' - data_fields.deployment_cloud! - else - data_fields.deployment_unknown! - end - end - - def self.event_description(event) - case event - when "merge_request", "merge_request_events" - s_("JiraService|Jira comments will be created when an issue gets referenced in a merge request.") - when "commit", "commit_events" - s_("JiraService|Jira comments will be created when an issue gets referenced in a commit.") - end - end -end - -JiraService.prepend_mod_with('JiraService') diff --git a/app/models/project_services/jira_tracker_data.rb b/app/models/project_services/jira_tracker_data.rb deleted file mode 100644 index 2c145abf5c9..00000000000 --- a/app/models/project_services/jira_tracker_data.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -class JiraTrackerData < ApplicationRecord - include Services::DataFields - include IgnorableColumns - - ignore_columns %i[ - encrypted_proxy_address - encrypted_proxy_address_iv - encrypted_proxy_port - encrypted_proxy_port_iv - encrypted_proxy_username - encrypted_proxy_username_iv - encrypted_proxy_password - encrypted_proxy_password_iv - ], remove_with: '14.0', remove_after: '2021-05-22' - - attr_encrypted :url, encryption_options - attr_encrypted :api_url, encryption_options - attr_encrypted :username, encryption_options - attr_encrypted :password, encryption_options - - enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment -end diff --git a/app/models/project_services/mattermost_service.rb b/app/models/project_services/mattermost_service.rb deleted file mode 100644 index 732a7c32a03..00000000000 --- a/app/models/project_services/mattermost_service.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class MattermostService < ChatNotificationService - include SlackMattermost::Notifier - include ActionView::Helpers::UrlHelper - - def title - s_('Mattermost notifications') - end - - def description - s_('Send notifications about project events to Mattermost channels.') - end - - def self.to_param - 'mattermost' - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/mattermost'), target: '_blank', rel: 'noopener noreferrer' - s_('Send notifications about project events to Mattermost channels. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def default_channel_placeholder - 'my-channel' - end - - def webhook_placeholder - 'http://mattermost.example.com/hooks/' - end -end diff --git a/app/models/project_services/mattermost_slash_commands_service.rb b/app/models/project_services/mattermost_slash_commands_service.rb deleted file mode 100644 index 60235a09dcd..00000000000 --- a/app/models/project_services/mattermost_slash_commands_service.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -class MattermostSlashCommandsService < SlashCommandsService - include Ci::TriggersHelper - - prop_accessor :token - - def can_test? - false - end - - def title - 'Mattermost slash commands' - end - - def description - "Perform common tasks with slash commands." - end - - def self.to_param - 'mattermost_slash_commands' - end - - def configure(user, params) - token = Mattermost::Command.new(user) - .create(command(params)) - - update(active: true, token: token) if token - rescue Mattermost::Error => e - [false, e.message] - end - - def list_teams(current_user) - [Mattermost::Team.new(current_user).all, nil] - rescue Mattermost::Error => e - [[], e.message] - end - - def chat_responder - ::Gitlab::Chat::Responder::Mattermost - end - - private - - def command(params) - pretty_project_name = project.full_name - - params.merge( - auto_complete: true, - auto_complete_desc: "Perform common operations on: #{pretty_project_name}", - auto_complete_hint: '[help]', - description: "Perform common operations on: #{pretty_project_name}", - display_name: "GitLab / #{pretty_project_name}", - method: 'P', - username: 'GitLab') - end -end diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb deleted file mode 100644 index 1d2067067da..00000000000 --- a/app/models/project_services/microsoft_teams_service.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -class MicrosoftTeamsService < ChatNotificationService - def title - 'Microsoft Teams notifications' - end - - def description - 'Send notifications about project events to Microsoft Teams.' - end - - def self.to_param - 'microsoft_teams' - end - - def help - '<p>Use this service to send notifications about events in GitLab projects to your Microsoft Teams channels. <a href="https://docs.gitlab.com/ee/user/project/integrations/microsoft_teams.html">How do I configure this integration?</a></p>' - end - - def webhook_placeholder - 'https://outlook.office.com/webhook/…' - end - - def event_field(event) - end - - def default_channel_placeholder - end - - def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push - pipeline wiki_page] - end - - def default_fields - [ - { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, - { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'If selected, successful pipelines do not trigger a notification event.' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } - ] - end - - private - - def notify(message, opts) - MicrosoftTeams::Notifier.new(webhook).ping( - title: message.project_name, - summary: message.summary, - activity: message.activity, - attachments: message.attachments - ) - end - - def custom_data(data) - super(data).merge(markdown: true) - end -end diff --git a/app/models/project_services/mock_ci_service.rb b/app/models/project_services/mock_ci_service.rb deleted file mode 100644 index bd6344c6e1a..00000000000 --- a/app/models/project_services/mock_ci_service.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -# For an example companion mocking service, see https://gitlab.com/gitlab-org/gitlab-mock-ci-service -class MockCiService < CiService - ALLOWED_STATES = %w[failed canceled running pending success success-with-warnings skipped not_found].freeze - - prop_accessor :mock_service_url - validates :mock_service_url, presence: true, public_url: true, if: :activated? - - def title - 'MockCI' - end - - def description - 'Mock an external CI' - end - - def self.to_param - 'mock_ci' - end - - def fields - [ - { - type: 'text', - name: 'mock_service_url', - title: s_('ProjectService|Mock service URL'), - placeholder: 'http://localhost:4004', - required: true - } - ] - end - - # Return complete url to build page - # - # Ex. - # http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c - # - def build_page(sha, ref) - Gitlab::Utils.append_path( - mock_service_url, - "#{project.namespace.path}/#{project.path}/status/#{sha}") - end - - # Return string with build status or :error symbol - # - # Allowed states: 'success', 'failed', 'running', 'pending', 'skipped' - # - # - # Ex. - # @service.commit_status('13be4ac', 'master') - # # => 'success' - # - # @service.commit_status('2abe4ac', 'dev') - # # => 'running' - # - # - def commit_status(sha, ref) - response = Gitlab::HTTP.get(commit_status_path(sha), verify: false) - read_commit_status(response) - rescue Errno::ECONNREFUSED - :error - end - - def commit_status_path(sha) - Gitlab::Utils.append_path( - mock_service_url, - "#{project.namespace.path}/#{project.path}/status/#{sha}.json") - end - - def read_commit_status(response) - return :error unless response.code == 200 || response.code == 404 - - status = if response.code == 404 - 'pending' - else - response['status'] - end - - if status.present? && ALLOWED_STATES.include?(status) - status - else - :error - end - end - - def can_test? - false - end -end diff --git a/app/models/project_services/open_project_service.rb b/app/models/project_services/open_project_service.rb deleted file mode 100644 index a24fbc1611d..00000000000 --- a/app/models/project_services/open_project_service.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class OpenProjectService < IssueTrackerService - validates :url, public_url: true, presence: true, if: :activated? - validates :api_url, public_url: true, allow_blank: true, if: :activated? - validates :token, presence: true, if: :activated? - validates :project_identifier_code, presence: true, if: :activated? - - data_field :url, :api_url, :token, :closed_status_id, :project_identifier_code - - def data_fields - open_project_tracker_data || self.build_open_project_tracker_data - end - - def self.to_param - 'open_project' - end -end diff --git a/app/models/project_services/open_project_tracker_data.rb b/app/models/project_services/open_project_tracker_data.rb deleted file mode 100644 index 20de60e40c1..00000000000 --- a/app/models/project_services/open_project_tracker_data.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class OpenProjectTrackerData < ApplicationRecord - include Services::DataFields - - # When the Open Project is fresh installed, the default closed status id is "13" based on current version: v8. - DEFAULT_CLOSED_STATUS_ID = "13" - - attr_encrypted :url, encryption_options - attr_encrypted :api_url, encryption_options - attr_encrypted :token, encryption_options - - def closed_status_id - super || DEFAULT_CLOSED_STATUS_ID - end -end diff --git a/app/models/project_services/packagist_service.rb b/app/models/project_services/packagist_service.rb deleted file mode 100644 index f3ea8c64302..00000000000 --- a/app/models/project_services/packagist_service.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -class PackagistService < Integration - prop_accessor :username, :token, :server - - validates :username, presence: true, if: :activated? - validates :token, presence: true, if: :activated? - - default_value_for :push_events, true - default_value_for :tag_push_events, true - - after_save :compose_service_hook, if: :activated? - - def title - 'Packagist' - end - - def description - s_('Integrations|Update your Packagist projects.') - end - - def self.to_param - 'packagist' - end - - def fields - [ - { type: 'text', name: 'username', placeholder: '', required: true }, - { type: 'text', name: 'token', placeholder: '', required: true }, - { type: 'text', name: 'server', placeholder: 'https://packagist.org', required: false } - ] - end - - def self.supported_events - %w(push merge_request tag_push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - service_hook.execute(data) - end - - def test(data) - begin - result = execute(data) - return { success: false, result: result[:message] } if result[:http_status] != 202 - rescue StandardError => error - return { success: false, result: error } - end - - { success: true, result: result[:message] } - end - - def compose_service_hook - hook = service_hook || build_service_hook - hook.url = hook_url - hook.save - end - - def hook_url - base_url = server.presence || 'https://packagist.org' - "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}" - end -end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb deleted file mode 100644 index 4603193ac8e..00000000000 --- a/app/models/project_services/pipelines_email_service.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -class PipelinesEmailService < Integration - include NotificationBranchSelection - - prop_accessor :recipients, :branches_to_be_notified - boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch - validates :recipients, presence: true, if: :validate_recipients? - - def initialize_properties - if properties.nil? - self.properties = {} - self.notify_only_broken_pipelines = true - self.branches_to_be_notified = "default" - elsif !self.notify_only_default_branch.nil? - # In older versions, there was only a boolean property named - # `notify_only_default_branch`. Now we have a string property named - # `branches_to_be_notified`. Instead of doing a background migration, we - # opted to set a value for the new property based on the old one, if - # users hasn't specified one already. When users edit the service and - # selects a value for this new property, it will override everything. - - self.branches_to_be_notified ||= notify_only_default_branch? ? "default" : "all" - end - end - - def title - _('Pipeline status emails') - end - - def description - _('Email the pipeline status to a list of recipients.') - end - - def self.to_param - 'pipelines_email' - end - - def self.supported_events - %w[pipeline] - end - - def self.default_test_event - 'pipeline' - end - - def execute(data, force: false) - return unless supported_events.include?(data[:object_kind]) - return unless force || should_pipeline_be_notified?(data) - - all_recipients = retrieve_recipients(data) - - return unless all_recipients.any? - - pipeline_id = data[:object_attributes][:id] - PipelineNotificationWorker.new.perform(pipeline_id, recipients: all_recipients) - end - - def can_test? - project&.ci_pipelines&.any? - end - - def fields - [ - { type: 'textarea', - name: 'recipients', - help: _('Comma-separated list of email addresses.'), - required: true }, - { type: 'checkbox', - name: 'notify_only_broken_pipelines' }, - { type: 'select', - name: 'branches_to_be_notified', - choices: branch_choices } - ] - end - - def test(data) - result = execute(data, force: true) - - { success: true, result: result } - rescue StandardError => error - { success: false, result: error } - end - - def should_pipeline_be_notified?(data) - notify_for_branch?(data) && notify_for_pipeline?(data) - end - - def notify_for_pipeline?(data) - case data[:object_attributes][:status] - when 'success' - !notify_only_broken_pipelines? - when 'failed' - true - else - false - end - end - - def retrieve_recipients(data) - recipients.to_s.split(/[,\r\n ]+/).reject(&:empty?) - end -end diff --git a/app/models/project_services/pivotaltracker_service.rb b/app/models/project_services/pivotaltracker_service.rb deleted file mode 100644 index 6e67984591d..00000000000 --- a/app/models/project_services/pivotaltracker_service.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -class PivotaltrackerService < Integration - API_ENDPOINT = 'https://www.pivotaltracker.com/services/v5/source_commits' - - prop_accessor :token, :restrict_to_branch - validates :token, presence: true, if: :activated? - - def title - 'PivotalTracker' - end - - def description - s_('PivotalTrackerService|Add commit messages as comments to PivotalTracker stories.') - end - - def self.to_param - 'pivotaltracker' - end - - def fields - [ - { - type: 'text', - name: 'token', - placeholder: s_('PivotalTrackerService|Pivotal Tracker API token.'), - required: true - }, - { - type: 'text', - name: 'restrict_to_branch', - placeholder: s_('PivotalTrackerService|Comma-separated list of branches which will be ' \ - 'automatically inspected. Leave blank to include all branches.') - } - ] - end - - def self.supported_events - %w(push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - return unless allowed_branch?(data[:ref]) - - data[:commits].each do |commit| - message = { - 'source_commit' => { - 'commit_id' => commit[:id], - 'author' => commit[:author][:name], - 'url' => commit[:url], - 'message' => commit[:message] - } - } - Gitlab::HTTP.post( - API_ENDPOINT, - body: message.to_json, - headers: { - 'Content-Type' => 'application/json', - 'X-TrackerToken' => token - } - ) - end - end - - private - - def allowed_branch?(ref) - return true unless ref.present? && restrict_to_branch.present? - - branch = Gitlab::Git.ref_name(ref) - allowed_branches = restrict_to_branch.split(',').map(&:strip) - - branch.present? && allowed_branches.include?(branch) - end -end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index b8869547a37..a289c1c2afb 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -117,8 +117,8 @@ class PrometheusService < MonitoringService return false if template? return false unless project - project.all_clusters.enabled.eager_load(:application_prometheus).any? do |cluster| - cluster.application_prometheus&.available? + project.all_clusters.enabled.eager_load(:integration_prometheus).any? do |cluster| + cluster.integration_prometheus_available? end end diff --git a/app/models/project_services/pushover_service.rb b/app/models/project_services/pushover_service.rb deleted file mode 100644 index 89765fbdf41..00000000000 --- a/app/models/project_services/pushover_service.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -class PushoverService < Integration - BASE_URI = 'https://api.pushover.net/1' - - prop_accessor :api_key, :user_key, :device, :priority, :sound - validates :api_key, :user_key, :priority, presence: true, if: :activated? - - def title - 'Pushover' - end - - def description - s_('PushoverService|Get real-time notifications on your device.') - end - - def self.to_param - 'pushover' - end - - def fields - [ - { type: 'text', name: 'api_key', title: _('API key'), placeholder: s_('PushoverService|Your application key'), required: true }, - { type: 'text', name: 'user_key', placeholder: s_('PushoverService|Your user key'), required: true }, - { type: 'text', name: 'device', placeholder: s_('PushoverService|Leave blank for all active devices') }, - { type: 'select', name: 'priority', required: true, choices: - [ - [s_('PushoverService|Lowest Priority'), -2], - [s_('PushoverService|Low Priority'), -1], - [s_('PushoverService|Normal Priority'), 0], - [s_('PushoverService|High Priority'), 1] - ], - default_choice: 0 }, - { type: 'select', name: 'sound', choices: - [ - ['Device default sound', nil], - ['Pushover (default)', 'pushover'], - %w(Bike bike), - %w(Bugle bugle), - ['Cash Register', 'cashregister'], - %w(Classical classical), - %w(Cosmic cosmic), - %w(Falling falling), - %w(Gamelan gamelan), - %w(Incoming incoming), - %w(Intermission intermission), - %w(Magic magic), - %w(Mechanical mechanical), - ['Piano Bar', 'pianobar'], - %w(Siren siren), - ['Space Alarm', 'spacealarm'], - ['Tug Boat', 'tugboat'], - ['Alien Alarm (long)', 'alien'], - ['Climb (long)', 'climb'], - ['Persistent (long)', 'persistent'], - ['Pushover Echo (long)', 'echo'], - ['Up Down (long)', 'updown'], - ['None (silent)', 'none'] - ] } - ] - end - - def self.supported_events - %w(push) - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - ref = Gitlab::Git.ref_name(data[:ref]) - before = data[:before] - after = data[:after] - - message = - if Gitlab::Git.blank_ref?(before) - s_("PushoverService|%{user_name} pushed new branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } - elsif Gitlab::Git.blank_ref?(after) - s_("PushoverService|%{user_name} deleted branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } - else - s_("PushoverService|%{user_name} push to branch \"%{ref}\".") % { user_name: data[:user_name], ref: ref } - end - - if data[:total_commits_count] > 0 - message = [message, s_("PushoverService|Total commits count: %{total_commits_count}") % { total_commits_count: data[:total_commits_count] }].join("\n") - end - - pushover_data = { - token: api_key, - user: user_key, - device: device, - priority: priority, - title: "#{project.full_name}", - message: message, - url: data[:project][:web_url], - url_title: s_("PushoverService|See project %{project_full_name}") % { project_full_name: project.full_name } - } - - # Sound parameter MUST NOT be sent to API if not selected - if sound - pushover_data[:sound] = sound - end - - Gitlab::HTTP.post('/messages.json', base_uri: BASE_URI, body: pushover_data) - end -end diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb deleted file mode 100644 index 7a0f500209c..00000000000 --- a/app/models/project_services/redmine_service.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class RedmineService < IssueTrackerService - include ActionView::Helpers::UrlHelper - validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - - def title - 'Redmine' - end - - def description - s_("IssueTracker|Use Redmine as this project's issue tracker.") - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/redmine'), target: '_blank', rel: 'noopener noreferrer' - s_('IssueTracker|Use Redmine as the issue tracker. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'redmine' - end -end diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb deleted file mode 100644 index 92a46f8d01f..00000000000 --- a/app/models/project_services/slack_service.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -class SlackService < ChatNotificationService - include SlackMattermost::Notifier - extend ::Gitlab::Utils::Override - - SUPPORTED_EVENTS_FOR_USAGE_LOG = %w[ - push issue confidential_issue merge_request note confidential_note - tag_push wiki_page deployment - ].freeze - - prop_accessor EVENT_CHANNEL['alert'] - - def title - 'Slack notifications' - end - - def description - 'Send notifications about project events to Slack.' - end - - def self.to_param - 'slack' - end - - def default_channel_placeholder - _('general, development') - end - - def webhook_placeholder - 'https://hooks.slack.com/services/…' - end - - def supported_events - additional = [] - additional << 'alert' - - super + additional - end - - def get_message(object_kind, data) - return Integrations::ChatMessage::AlertMessage.new(data) if object_kind == 'alert' - - super - end - - override :log_usage - def log_usage(event, user_id) - return unless user_id - - return unless SUPPORTED_EVENTS_FOR_USAGE_LOG.include?(event) - - key = "i_ecosystem_slack_service_#{event}_notification" - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) - end -end diff --git a/app/models/project_services/slack_slash_commands_service.rb b/app/models/project_services/slack_slash_commands_service.rb deleted file mode 100644 index 548f3623504..00000000000 --- a/app/models/project_services/slack_slash_commands_service.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -class SlackSlashCommandsService < SlashCommandsService - include Ci::TriggersHelper - - def title - 'Slack slash commands' - end - - def description - "Perform common operations in Slack" - end - - def self.to_param - 'slack_slash_commands' - end - - def trigger(params) - # Format messages to be Slack-compatible - super.tap do |result| - result[:text] = format(result[:text]) if result.is_a?(Hash) - end - end - - def chat_responder - ::Gitlab::Chat::Responder::Slack - end - - private - - def format(text) - Slack::Messenger::Util::LinkFormatter.format(text) if text - end -end diff --git a/app/models/project_services/slash_commands_service.rb b/app/models/project_services/slash_commands_service.rb deleted file mode 100644 index 37d16737052..00000000000 --- a/app/models/project_services/slash_commands_service.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# Base class for Chat services -# This class is not meant to be used directly, but only to inherrit from. -class SlashCommandsService < Integration - default_value_for :category, 'chat' - - prop_accessor :token - - has_many :chat_names, foreign_key: :service_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - - def valid_token?(token) - self.respond_to?(:token) && - self.token.present? && - ActiveSupport::SecurityUtils.secure_compare(token, self.token) - end - - def self.supported_events - %w() - end - - def can_test? - false - end - - def fields - [ - { type: 'text', name: 'token', placeholder: 'XXxxXXxxXXxxXXxxXXxxXXxx' } - ] - end - - def trigger(params) - return unless valid_token?(params[:token]) - - chat_user = find_chat_user(params) - user = chat_user&.user - - if user - unless user.can?(:use_slash_commands) - return Gitlab::SlashCommands::Presenters::Access.new.deactivated if user.deactivated? - - return Gitlab::SlashCommands::Presenters::Access.new.access_denied(project) - end - - Gitlab::SlashCommands::Command.new(project, chat_user, params).execute - else - url = authorize_chat_name_url(params) - Gitlab::SlashCommands::Presenters::Access.new(url).authorize - end - end - - private - - # rubocop: disable CodeReuse/ServiceClass - def find_chat_user(params) - ChatNames::FindUserService.new(self, params).execute - end - # rubocop: enable CodeReuse/ServiceClass - - # rubocop: disable CodeReuse/ServiceClass - def authorize_chat_name_url(params) - ChatNames::AuthorizeUserService.new(self, params).execute - end - # rubocop: enable CodeReuse/ServiceClass -end diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb deleted file mode 100644 index 6fc24a4778c..00000000000 --- a/app/models/project_services/teamcity_service.rb +++ /dev/null @@ -1,189 +0,0 @@ -# frozen_string_literal: true - -class TeamcityService < CiService - include ReactiveService - include ServicePushDataValidations - - prop_accessor :teamcity_url, :build_type, :username, :password - - validates :teamcity_url, presence: true, public_url: true, if: :activated? - validates :build_type, presence: true, if: :activated? - validates :username, - presence: true, - if: ->(service) { service.activated? && service.password } - validates :password, - presence: true, - if: ->(service) { service.activated? && service.username } - - attr_accessor :response - - after_save :compose_service_hook, if: :activated? - before_update :reset_password - - class << self - def to_param - 'teamcity' - end - - def supported_events - %w(push merge_request) - end - - def event_description(event) - case event - when 'push', 'push_events' - 'TeamCity CI will be triggered after every push to the repository except branch delete' - when 'merge_request', 'merge_request_events' - 'TeamCity CI will be triggered after a merge request has been created or updated' - end - end - end - - def compose_service_hook - hook = service_hook || build_service_hook - hook.save - end - - def reset_password - if teamcity_url_changed? && !password_touched? - self.password = nil - end - end - - def title - 'JetBrains TeamCity' - end - - def description - s_('ProjectService|Run CI/CD pipelines with JetBrains TeamCity.') - end - - def help - s_('To run CI/CD pipelines with JetBrains TeamCity, input the GitLab project details in the TeamCity project Version Control Settings.') - end - - def fields - [ - { - type: 'text', - name: 'teamcity_url', - title: s_('ProjectService|TeamCity server URL'), - placeholder: 'https://teamcity.example.com', - required: true - }, - { - type: 'text', - name: 'build_type', - help: s_('ProjectService|The build configuration ID of the TeamCity project.'), - required: true - }, - { - type: 'text', - name: 'username', - help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') - }, - { - type: 'password', - name: 'password', - non_empty_password_title: s_('ProjectService|Enter new password'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password') - } - ] - end - - def build_page(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:build_page] } - end - - def commit_status(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } - end - - def calculate_reactive_cache(sha, ref) - response = get_path("httpAuth/app/rest/builds/branch:unspecified:any,revision:#{sha}") - - if response - { build_page: read_build_page(response), commit_status: read_commit_status(response) } - else - { build_page: teamcity_url, commit_status: :error } - end - end - - def execute(data) - case data[:object_kind] - when 'push' - execute_push(data) - when 'merge_request' - execute_merge_request(data) - end - end - - private - - def execute_push(data) - branch = Gitlab::Git.ref_name(data[:ref]) - post_to_build_queue(data, branch) if push_valid?(data) - end - - def execute_merge_request(data) - branch = data[:object_attributes][:source_branch] - post_to_build_queue(data, branch) if merge_request_valid?(data) - end - - def read_build_page(response) - if response.code != 200 - # If actual build link can't be determined, - # send user to build summary page. - build_url("viewLog.html?buildTypeId=#{build_type}") - else - # If actual build link is available, go to build result page. - built_id = response['build']['id'] - build_url("viewLog.html?buildId=#{built_id}&buildTypeId=#{build_type}") - end - end - - def read_commit_status(response) - return :error unless response.code == 200 || response.code == 404 - - status = if response.code == 404 - 'Pending' - else - response['build']['status'] - end - - return :error unless status.present? - - if status.include?('SUCCESS') - 'success' - elsif status.include?('FAILURE') - 'failed' - elsif status.include?('Pending') - 'pending' - else - :error - end - end - - def build_url(path) - Gitlab::Utils.append_path(teamcity_url, path) - end - - def get_path(path) - Gitlab::HTTP.try_get(build_url(path), verify: false, basic_auth: basic_auth, extra_log_info: { project_id: project_id }) - end - - def post_to_build_queue(data, branch) - Gitlab::HTTP.post( - build_url('httpAuth/app/rest/buildQueue'), - body: "<build branchName=#{branch.encode(xml: :attr)}>"\ - "<buildType id=#{build_type.encode(xml: :attr)}/>"\ - '</build>', - headers: { 'Content-type' => 'application/xml' }, - basic_auth: basic_auth - ) - end - - def basic_auth - { username: username, password: password } - end -end diff --git a/app/models/project_services/unify_circuit_service.rb b/app/models/project_services/unify_circuit_service.rb deleted file mode 100644 index 5f43388e1c9..00000000000 --- a/app/models/project_services/unify_circuit_service.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -class UnifyCircuitService < ChatNotificationService - def title - 'Unify Circuit' - end - - def description - s_('Integrations|Send notifications about project events to Unify Circuit.') - end - - def self.to_param - 'unify_circuit' - end - - def help - 'This service sends notifications about projects events to a Unify Circuit conversation.<br /> - To set up this service: - <ol> - <li><a href="https://www.circuit.com/unifyportalfaqdetail?articleId=164448">Set up an incoming webhook for your conversation</a>. All notifications will come to this conversation.</li> - <li>Paste the <strong>Webhook URL</strong> into the field below.</li> - <li>Select events below to enable notifications.</li> - </ol>' - end - - def event_field(event) - end - - def default_channel_placeholder - end - - def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push - pipeline wiki_page] - end - - def default_fields - [ - { type: 'text', name: 'webhook', placeholder: "e.g. https://circuit.com/rest/v2/webhooks/incoming/…", required: true }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } - ] - end - - private - - def notify(message, opts) - response = Gitlab::HTTP.post(webhook, body: { - subject: message.project_name, - text: message.summary, - markdown: true - }.to_json) - - response if response.success? - end - - def custom_data(data) - super(data).merge(markdown: true) - end -end diff --git a/app/models/project_services/webex_teams_service.rb b/app/models/project_services/webex_teams_service.rb deleted file mode 100644 index 3d92d3bb85e..00000000000 --- a/app/models/project_services/webex_teams_service.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -class WebexTeamsService < ChatNotificationService - include ActionView::Helpers::UrlHelper - - def title - s_("WebexTeamsService|Webex Teams") - end - - def description - s_("WebexTeamsService|Send notifications about project events to Webex Teams.") - end - - def self.to_param - 'webex_teams' - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/webex_teams'), target: '_blank', rel: 'noopener noreferrer' - s_("WebexTeamsService|Send notifications about project events to a Webex Teams conversation. %{docs_link}") % { docs_link: docs_link.html_safe } - end - - def event_field(event) - end - - def default_channel_placeholder - end - - def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push - pipeline wiki_page] - end - - def default_fields - [ - { type: 'text', name: 'webhook', placeholder: "https://api.ciscospark.com/v1/webhooks/incoming/...", required: true }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } - ] - end - - private - - def notify(message, opts) - header = { 'Content-Type' => 'application/json' } - response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.summary }.to_json) - - response if response.success? - end - - def custom_data(data) - super(data).merge(markdown: true) - end -end diff --git a/app/models/project_services/youtrack_service.rb b/app/models/project_services/youtrack_service.rb deleted file mode 100644 index 9760a22a872..00000000000 --- a/app/models/project_services/youtrack_service.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -class YoutrackService < IssueTrackerService - include ActionView::Helpers::UrlHelper - - validates :project_url, :issues_url, presence: true, public_url: true, if: :activated? - - # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030 - def self.reference_pattern(only_long: false) - if only_long - /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)/ - else - /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)|(#{Issue.reference_prefix}#{Gitlab::Regex.issue})/ - end - end - - def title - 'YouTrack' - end - - def description - s_("IssueTracker|Use YouTrack as this project's issue tracker.") - end - - def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/youtrack'), target: '_blank', rel: 'noopener noreferrer' - s_("IssueTracker|Use YouTrack as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } - end - - def self.to_param - 'youtrack' - end - - def fields - [ - { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in YouTrack.'), required: true }, - { type: 'text', name: 'issues_url', title: s_('ProjectService|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the YouTrack project. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true } - ] - end -end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 37ddd2d030d..387732cf151 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -94,18 +94,14 @@ class ProjectStatistics < ApplicationRecord end def update_storage_size - storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size - # The `snippets_size` column was added on 20200622095419 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb - # might try to update project statistics before the `snippets_size` column has been created. - storage_size += snippets_size if self.class.column_names.include?('snippets_size') - - # The `pipeline_artifacts_size` column was added on 20200817142800 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb - # might try to update project statistics before the `pipeline_artifacts_size` column has been created. - storage_size += pipeline_artifacts_size if self.class.column_names.include?('pipeline_artifacts_size') - - # The `uploads_size` column was added on 20201105021637 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb - # might try to update project statistics before the `uploads_size` column has been created. - storage_size += uploads_size if self.class.column_names.include?('uploads_size') + storage_size = repository_size + + wiki_size + + lfs_objects_size + + build_artifacts_size + + packages_size + + snippets_size + + pipeline_artifacts_size + + uploads_size self.storage_size = storage_size end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 889eaed138d..3df8fe31826 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -30,8 +30,6 @@ class ProtectedBranch < ApplicationRecord end def self.allow_force_push?(project, ref_name) - return false unless ::Feature.enabled?(:allow_force_push_to_protected_branches, project, default_enabled: :yaml) - project.protected_branches.allowing_force_push.matching(ref_name).any? end diff --git a/app/models/release.rb b/app/models/release.rb index 1889a0707b4..aad1cbeabdb 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -39,10 +39,10 @@ class Release < ApplicationRecord scope :released_within_2hrs, -> { where(released_at: Time.zone.now - 1.hour..Time.zone.now + 1.hour) } # Sorting - scope :order_created, -> { reorder('created_at ASC') } - scope :order_created_desc, -> { reorder('created_at DESC') } - scope :order_released, -> { reorder('released_at ASC') } - scope :order_released_desc, -> { reorder('released_at DESC') } + scope :order_created, -> { reorder(created_at: :asc) } + scope :order_created_desc, -> { reorder(created_at: :desc) } + scope :order_released, -> { reorder(released_at: :asc) } + scope :order_released_desc, -> { reorder(released_at: :desc) } delegate :repository, to: :project diff --git a/app/models/release_highlight.rb b/app/models/release_highlight.rb index 9c30d0611e6..84e0a43670b 100644 --- a/app/models/release_highlight.rb +++ b/app/models/release_highlight.rb @@ -33,7 +33,7 @@ class ReleaseHighlight next unless include_item?(item) begin - item.tap {|i| i['body'] = Kramdown::Document.new(i['body']).to_html } + item.tap {|i| i['body'] = Banzai.render(i['body'], { project: nil }) } rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, file_path: file_path) diff --git a/app/models/repository.rb b/app/models/repository.rb index 7dca8e52403..1bd61fe48cb 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -938,6 +938,8 @@ class Repository end def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil, prune: true) + return fetch_remote(remote_name, url: url, refmap: refmap, forced: forced, prune: prune) if Feature.enabled?(:fetch_remote_params, project, default_enabled: :yaml) + unless remote_name remote_name = "tmp-#{SecureRandom.hex}" tmp_remote_name = true diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb index bcc17d32272..c5203354b9d 100644 --- a/app/models/service_desk_setting.rb +++ b/app/models/service_desk_setting.rb @@ -6,9 +6,12 @@ class ServiceDeskSetting < ApplicationRecord belongs_to :project validates :project_id, presence: true validate :valid_issue_template + validate :valid_project_key validates :outgoing_name, length: { maximum: 255 }, allow_blank: true validates :project_key, length: { maximum: 255 }, allow_blank: true, format: { with: /\A[a-z0-9_]+\z/ } + scope :with_project_key, ->(key) { where(project_key: key) } + def issue_template_content strong_memoize(:issue_template_content) do next unless issue_template_key.present? @@ -27,4 +30,23 @@ class ServiceDeskSetting < ApplicationRecord errors.add(:issue_template_key, 'is empty or does not exist') end end + + def valid_project_key + if projects_with_same_slug_and_key_exists? + errors.add(:project_key, 'already in use for another service desk address.') + end + end + + private + + def projects_with_same_slug_and_key_exists? + return false unless project_key + + settings = self.class.with_project_key(project_key).preload(:project) + project_slug = self.project.full_path_slug + + settings.any? do |setting| + setting.project.full_path_slug == project_slug + end + end end diff --git a/app/models/snippet_repository_storage_move.rb b/app/models/snippet_repository_storage_move.rb deleted file mode 100644 index 8234905a7e1..00000000000 --- a/app/models/snippet_repository_storage_move.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -# This is a compatibility class to avoid calling a non-existent -# class from sidekiq during deployment. -# -# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. -# we cannot remove this class entirely because there can be jobs -# referencing it. -# -# We can get rid of this class in 14.0 -# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 -class SnippetRepositoryStorageMove < Snippets::RepositoryStorageMove -end diff --git a/app/models/timelog.rb b/app/models/timelog.rb index bd543526685..96fd485b797 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -18,8 +18,12 @@ class Timelog < ApplicationRecord joins(:project).where(projects: { namespace: group.self_and_descendants }) end - scope :between_times, -> (start_time, end_time) do - where('spent_at BETWEEN ? AND ?', start_time, end_time) + scope :at_or_after, -> (start_time) do + where('spent_at >= ?', start_time) + end + + scope :at_or_before, -> (end_time) do + where('spent_at <= ?', end_time) end def issuable diff --git a/app/models/todo.rb b/app/models/todo.rb index 23685fb68e0..94a99603848 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -61,6 +61,7 @@ class Todo < ApplicationRecord scope :for_author, -> (author) { where(author: author) } scope :for_user, -> (user) { where(user: user) } scope :for_project, -> (projects) { where(project: projects) } + scope :for_note, -> (notes) { where(note: notes) } scope :for_undeleted_projects, -> { joins(:project).merge(Project.without_deleted) } scope :for_group, -> (group) { where(group: group) } scope :for_type, -> (type) { where(target_type: type) } diff --git a/app/models/user.rb b/app/models/user.rb index 0eb58baae11..8ee0421e45f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -35,6 +35,9 @@ class User < ApplicationRecord COUNT_CACHE_VALIDITY_PERIOD = 24.hours + MAX_USERNAME_LENGTH = 255 + MIN_USERNAME_LENGTH = 2 + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token add_authentication_token_field :static_object_token @@ -96,12 +99,6 @@ class User < ApplicationRecord # Virtual attribute for impersonator attr_accessor :impersonator - attr_writer :max_access_for_group - - def max_access_for_group - @max_access_for_group ||= {} - end - # # Relations # @@ -111,7 +108,7 @@ class User < ApplicationRecord # Profile has_many :keys, -> { regular_keys }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :expired_today_and_unnotified_keys, -> { expired_today_and_not_notified }, class_name: 'Key' + has_many :expired_and_unnotified_keys, -> { expired_and_not_notified }, class_name: 'Key' has_many :expiring_soon_and_unnotified_keys, -> { expiring_soon_and_not_notified }, class_name: 'Key' has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :group_deploy_keys @@ -315,10 +312,11 @@ class User < ApplicationRecord delegate :other_role, :other_role=, to: :user_detail, allow_nil: true delegate :bio, :bio=, :bio_html, to: :user_detail, allow_nil: true delegate :webauthn_xid, :webauthn_xid=, to: :user_detail, allow_nil: true + delegate :pronouns, :pronouns=, to: :user_detail, allow_nil: true accepts_nested_attributes_for :user_preference, update_only: true accepts_nested_attributes_for :user_detail, update_only: true - accepts_nested_attributes_for :credit_card_validation, update_only: true + accepts_nested_attributes_for :credit_card_validation, update_only: true, allow_destroy: true state_machine :state, initial: :active do event :block do @@ -414,14 +412,7 @@ class User < ApplicationRecord .without_impersonation .expired_today_and_not_notified) end - scope :with_ssh_key_expired_today, -> do - includes(:expired_today_and_unnotified_keys) - .where('EXISTS (?)', - ::Key - .select(1) - .where('keys.user_id = users.id') - .expired_today_and_not_notified) - end + scope :with_ssh_key_expiring_soon, -> do includes(:expiring_soon_and_unnotified_keys) .where('EXISTS (?)', @@ -791,6 +782,16 @@ class User < ApplicationRecord end end + def automation_bot + email_pattern = "automation%s@#{Settings.gitlab.host}" + + unique_internal(where(user_type: :automation_bot), 'automation-bot', email_pattern) do |u| + u.bio = 'The GitLab automation bot used for automated workflows and tasks' + u.name = 'GitLab Automation Bot' + u.avatar = bot_avatar(image: 'support-bot.png') # todo: add an avatar for automation-bot + end + end + # Return true if there is only single non-internal user in the deployment, # ghost user is ignored. def single_user? @@ -1703,12 +1704,6 @@ class User < ApplicationRecord def invalidate_issue_cache_counts Rails.cache.delete(['users', id, 'assigned_open_issues_count']) - - if Feature.enabled?(:assigned_open_issues_cache, default_enabled: :yaml) - run_after_commit do - Users::UpdateOpenIssueCountWorker.perform_async(self.id) - end - end end def invalidate_merge_request_cache_counts @@ -1928,6 +1923,20 @@ class User < ApplicationRecord confirmed? && !blocked? && !ghost? end + # This attribute hosts a Ci::JobToken::Scope object which is set when + # the user is authenticated successfully via CI_JOB_TOKEN. + def ci_job_token_scope + Gitlab::SafeRequestStore[ci_job_token_scope_cache_key] + end + + def set_ci_job_token_scope!(job) + Gitlab::SafeRequestStore[ci_job_token_scope_cache_key] = Ci::JobToken::Scope.new(job.project) + end + + def from_ci_job_token? + ci_job_token_scope.present? + end + protected # override, from Devise::Validatable @@ -2091,6 +2100,10 @@ class User < ApplicationRecord def update_highest_role_attribute id end + + def ci_job_token_scope_cache_key + "users:#{id}:ci:job_token_scope" + end end User.prepend_mod_with('User') diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index 8fc9efddac9..2e8ff1b7b49 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -16,9 +16,7 @@ class UserCallout < ApplicationRecord tabs_position_highlight: 10, threat_monitoring_info: 11, # EE-only account_recovery_regular_check: 12, # EE-only - webhooks_moved: 13, service_templates_deprecated_callout: 14, - admin_integrations_moved: 15, web_ide_alert_dismissed: 16, # no longer in use active_user_count_threshold: 18, # EE-only buy_pipeline_minutes_notification_dot: 19, # EE-only @@ -32,7 +30,8 @@ class UserCallout < ApplicationRecord eoa_bronze_plan_banner: 28, # EE-only pipeline_needs_banner: 29, pipeline_needs_hover_tip: 30, - web_ide_ci_environments_guidance: 31 + web_ide_ci_environments_guidance: 31, + security_configuration_upgrade_banner: 32 } validates :user, presence: true diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 458764632ed..47537e5885f 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -6,6 +6,7 @@ class UserDetail < ApplicationRecord belongs_to :user + validates :pronouns, length: { maximum: 50 } validates :job_title, length: { maximum: 200 } validates :bio, length: { maximum: 255 }, allow_blank: true diff --git a/app/models/users/in_product_marketing_email.rb b/app/models/users/in_product_marketing_email.rb index 195cfe162ac..3e5e7b259d8 100644 --- a/app/models/users/in_product_marketing_email.rb +++ b/app/models/users/in_product_marketing_email.rb @@ -18,7 +18,8 @@ module Users create: 0, verify: 1, trial: 2, - team: 3 + team: 3, + experience: 4 }, _suffix: true scope :without_track_and_series, -> (track, series) do |