diff options
Diffstat (limited to 'app/models')
95 files changed, 2984 insertions, 1247 deletions
diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index b01a244032d..2340453831e 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -1,4 +1,8 @@ class AbuseReport < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :message, pipeline: :single_line + belongs_to :reporter, class_name: 'User' belongs_to :user @@ -7,6 +11,9 @@ class AbuseReport < ActiveRecord::Base validates :message, presence: true validates :user_id, uniqueness: { message: 'has already been reported' } + # For CacheMarkdownField + alias_method :author, :reporter + def remove_user(deleted_by:) user.block DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true) diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 4cf8dd9a8ce..e4106e1c2e9 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,4 +1,8 @@ class Appearance < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :description + validates :title, presence: true validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 55d2e07de08..bf463a3b6bb 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,5 +1,7 @@ class ApplicationSetting < ActiveRecord::Base + include CacheMarkdownField include TokenAuthenticatable + add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token @@ -16,6 +18,13 @@ class ApplicationSetting < ActiveRecord::Base serialize :disabled_oauth_sign_in_sources, Array serialize :domain_whitelist, Array serialize :domain_blacklist, Array + serialize :repository_storages + serialize :sidekiq_throttling_queues, Array + + cache_markdown_field :sign_in_text + cache_markdown_field :help_page_text + cache_markdown_field :shared_runners_text, pipeline: :plain_markdown + cache_markdown_field :after_sign_up_text attr_accessor :domain_whitelist_raw, :domain_blacklist_raw @@ -67,9 +76,8 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: 0 } - validates :repository_storage, - presence: true, - inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + validates :repository_storages, presence: true + validate :check_repository_storages validates :enabled_git_access_protocol, inclusion: { in: %w(ssh http), allow_blank: true, allow_nil: true } @@ -78,6 +86,27 @@ class ApplicationSetting < ActiveRecord::Base presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' }, if: :domain_blacklist_enabled? + validates :sidekiq_throttling_factor, + numericality: { greater_than: 0, less_than: 1 }, + presence: { message: 'Throttling factor cannot be empty if Sidekiq Throttling is enabled.' }, + if: :sidekiq_throttling_enabled? + + validates :sidekiq_throttling_queues, + presence: { message: 'Queues to throttle cannot be empty if Sidekiq Throttling is enabled.' }, + if: :sidekiq_throttling_enabled? + + validates :housekeeping_incremental_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: 0 } + + validates :housekeeping_full_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_incremental_repack_period } + + validates :housekeeping_gc_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_full_repack_period } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -159,8 +188,14 @@ class ApplicationSetting < ActiveRecord::Base disabled_oauth_sign_in_sources: [], send_user_confirmation_email: false, container_registry_token_expire_delay: 5, - repository_storage: 'default', + repository_storages: ['default'], user_default_external: false, + sidekiq_throttling_enabled: false, + housekeeping_enabled: true, + housekeeping_bitmaps_enabled: true, + housekeeping_incremental_repack_period: 10, + housekeeping_full_repack_period: 50, + housekeeping_gc_period: 200, ) end @@ -168,6 +203,10 @@ class ApplicationSetting < ActiveRecord::Base ActiveRecord::Base.connection.column_exists?(:application_settings, :home_page_url) end + def sidekiq_throttling_column_exists? + ActiveRecord::Base.connection.column_exists?(:application_settings, :sidekiq_throttling_enabled) + end + def domain_whitelist_raw self.domain_whitelist.join("\n") unless self.domain_whitelist.nil? end @@ -194,6 +233,25 @@ class ApplicationSetting < ActiveRecord::Base self.domain_blacklist_raw = file.read end + def repository_storages + Array(read_attribute(:repository_storages)) + end + + # repository_storage is still required in the API. Remove in 9.0 + def repository_storage + repository_storages.first + end + + def repository_storage=(value) + self.repository_storages = [value] + end + + # Choose one of the available repository storage options. Currently all have + # equal weighting. + def pick_repository_storage + repository_storages.sample + end + def runners_registration_token ensure_runners_registration_token! end @@ -201,4 +259,18 @@ class ApplicationSetting < ActiveRecord::Base def health_check_access_token ensure_health_check_access_token! end + + def sidekiq_throttling_enabled? + return false unless sidekiq_throttling_column_exists? + + sidekiq_throttling_enabled + end + + private + + def check_repository_storages + invalid = repository_storages - Gitlab.config.repositories.storages.keys + errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless + invalid.empty? + end end diff --git a/app/models/board.rb b/app/models/board.rb index 3240c4bede3..c56422914a9 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -4,4 +4,12 @@ class Board < ActiveRecord::Base has_many :lists, -> { order(:list_type, :position) }, dependent: :delete_all validates :project, presence: true + + def backlog_list + lists.merge(List.backlog).take + end + + def done_list + lists.merge(List.done).take + end end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 61498140f27..cb40f33932a 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -1,6 +1,9 @@ class BroadcastMessage < ActiveRecord::Base + include CacheMarkdownField include Sortable + cache_markdown_field :message, pipeline: :broadcast_message + validates :message, presence: true validates :starts_at, presence: true validates :ends_at, presence: true diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb new file mode 100644 index 00000000000..f321db75eeb --- /dev/null +++ b/app/models/chat_name.rb @@ -0,0 +1,12 @@ +class ChatName < ActiveRecord::Base + belongs_to :service + belongs_to :user + + validates :user, presence: true + validates :service, presence: true + validates :team_id, presence: true + validates :chat_id, presence: true + + validates :user_id, uniqueness: { scope: [:service_id] } + validates :chat_id, uniqueness: { scope: [:service_id, :team_id] } +end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index fb16bc06d71..e7d33bd26db 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,9 +1,14 @@ module Ci class Build < CommitStatus - belongs_to :runner, class_name: 'Ci::Runner' - belongs_to :trigger_request, class_name: 'Ci::TriggerRequest' + include TokenAuthenticatable + include AfterCommitQueue + + belongs_to :runner + belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' + has_many :deployments, as: :deployable + serialize :options serialize :yaml_variables @@ -23,7 +28,10 @@ module Ci acts_as_taggable + add_authentication_token_field :token + before_save :update_artifacts_size, if: :artifacts_file_changed? + before_save :ensure_token before_destroy { project } after_create :execute_hooks @@ -38,6 +46,7 @@ module Ci new_build.status = 'pending' new_build.runner_id = nil new_build.trigger_request_id = nil + new_build.token = nil new_build.save end @@ -61,7 +70,11 @@ module Ci environment: build.environment, status_event: 'enqueue' ) - MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build) + + MergeRequests::AddTodoWhenBuildFailsService + .new(build.project, nil) + .close(new_build) + build.pipeline.mark_as_processable_after_stage(build.stage_idx) new_build end @@ -69,22 +82,20 @@ module Ci state_machine :status do after_transition pending: :running do |build| - build.execute_hooks + build.run_after_commit do + BuildHooksWorker.perform_async(id) + end end after_transition any => [:success, :failed, :canceled] do |build| - build.update_coverage - build.execute_hooks + build.run_after_commit do + BuildFinishedWorker.perform_async(id) + end end after_transition any => [:success] do |build| - if build.environment.present? - service = CreateDeploymentService.new(build.project, build.user, - environment: build.environment, - sha: build.sha, - ref: build.ref, - tag: build.tag) - service.execute(build) + build.run_after_commit do + BuildSuccessWorker.perform_async(id) end end end @@ -120,6 +131,34 @@ module Ci !self.pipeline.statuses.latest.include?(self) end + def expanded_environment_name + ExpandVariables.expand(environment, variables) if environment + end + + def has_environment? + self.environment.present? + end + + def starts_environment? + has_environment? && self.environment_action == 'start' + end + + def stops_environment? + has_environment? && self.environment_action == 'stop' + end + + def environment_action + self.options.fetch(:environment, {}).fetch(:action, 'start') + end + + def outdated_deployment? + success? && !last_deployment.try(:last?) + end + + def last_deployment + deployments.last + end + def depends_on_builds # Get builds of the same type latest_builds = self.pipeline.builds.latest @@ -128,13 +167,17 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def trace_html - trace_with_state[:html] || '' + def trace_html(**args) + trace_with_state(**args)[:html] || '' end - def trace_with_state(state = nil) - trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present? - trace_with_state || {} + def trace_with_state(state: nil, last_lines: nil) + trace_ansi = trace(last_lines: last_lines) + if trace_ansi.present? + Ci::Ansi2html.convert(trace_ansi, state) + else + {} + end end def timeout @@ -173,7 +216,7 @@ module Ci end def repo_url - auth = "gitlab-ci-token:#{token}@" + auth = "gitlab-ci-token:#{ensure_token!}@" project.http_url_to_repo.sub(/^https?:\/\//) do |prefix| prefix + auth end @@ -217,9 +260,10 @@ module Ci raw_trace.present? end - def raw_trace + def raw_trace(last_lines: nil) if File.exist?(trace_file_path) - File.read(trace_file_path) + Gitlab::Ci::TraceReader.new(trace_file_path). + read(last_lines: last_lines) else # backward compatibility read_attribute :trace @@ -234,13 +278,8 @@ module Ci project.ci_id && File.exist?(old_path_to_trace) end - def trace - trace = raw_trace - if project && trace.present? && project.runners_token.present? - trace.gsub(project.runners_token, 'xxxxxx') - else - trace - end + def trace(last_lines: nil) + hide_secrets(raw_trace(last_lines: last_lines)) end def trace_length @@ -253,6 +292,7 @@ module Ci def trace=(trace) recreate_trace_dir + trace = hide_secrets(trace) File.write(path_to_trace, trace) end @@ -265,6 +305,9 @@ module Ci def append_trace(trace_part, offset) recreate_trace_dir + touch if needs_touch? + + trace_part = hide_secrets(trace_part) File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) File.open(path_to_trace, 'ab') do |f| @@ -272,6 +315,10 @@ module Ci end end + def needs_touch? + Time.now - updated_at > 15.minutes.to_i + end + def trace_file_path if has_old_trace_file? old_path_to_trace @@ -341,12 +388,8 @@ module Ci ) end - def token - project.runners_token - end - def valid_token?(token) - project.valid_runners_token?(token) + self.token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) end def has_tags? @@ -370,7 +413,7 @@ module Ci end def artifacts? - !artifacts_expired? && self[:artifacts_file].present? + !artifacts_expired? && artifacts_file.exists? end def artifacts_metadata? @@ -444,6 +487,10 @@ module Ci ] end + def credentials + Gitlab::Ci::Build::Credentials::Factory.new(self).create! + end + private def update_artifacts_size @@ -488,5 +535,14 @@ module Ci pipeline.config_processor.build_attributes(name) end + + def hide_secrets(trace) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) + trace + end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 0b1df9f4294..3fee6c18770 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -2,22 +2,24 @@ module Ci class Pipeline < ActiveRecord::Base extend Ci::Model include HasStatus + include Importable + include AfterCommitQueue self.table_name = 'ci_commits' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :project, foreign_key: :gl_project_id belongs_to :user has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id - has_many :builds, class_name: 'Ci::Build', foreign_key: :commit_id - has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest', foreign_key: :commit_id + has_many :builds, foreign_key: :commit_id + has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id - validates_presence_of :sha - validates_presence_of :ref - validates_presence_of :status - validate :valid_commit_sha + validates_presence_of :sha, unless: :importing? + validates_presence_of :ref, unless: :importing? + validates_presence_of :status, unless: :importing? + validate :valid_commit_sha, unless: :importing? - after_save :keep_around_commits + after_create :keep_around_commits, unless: :importing? delegate :stages, to: :statuses @@ -28,39 +30,62 @@ module Ci end event :run do - transition any => :running + transition any - [:running] => :running end event :skip do - transition any => :skipped + transition any - [:skipped] => :skipped end event :drop do - transition any => :failed + transition any - [:failed] => :failed end event :succeed do - transition any => :success + transition any - [:success] => :success end event :cancel do - transition any => :canceled + transition any - [:canceled] => :canceled end + # IMPORTANT + # Do not add any operations to this state_machine + # Create a separate worker for each new operation + before_transition [:created, :pending] => :running do |pipeline| pipeline.started_at = Time.now end before_transition any => [:success, :failed, :canceled] do |pipeline| pipeline.finished_at = Time.now + pipeline.update_duration end - before_transition do |pipeline| - pipeline.update_duration + after_transition [:created, :pending] => :running do |pipeline| + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + end + + after_transition any => [:success] do |pipeline| + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + end + + after_transition [:created, :pending, :running] => :success do |pipeline| + pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) } end after_transition do |pipeline, transition| - pipeline.execute_hooks unless transition.loopback? + next if transition.loopback? + + pipeline.run_after_commit do + PipelineHooksWorker.perform_async(id) + end + end + + after_transition any => [:success, :failed] do |pipeline| + pipeline.run_after_commit do + PipelineNotificationWorker.perform_async(pipeline.id) + end end end @@ -90,6 +115,11 @@ module Ci project.id end + # For now the only user who participates is the user who triggered + def participants(_current_user = nil) + Array(user) + end + def valid_commit_sha if self.sha == Gitlab::Git::BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") @@ -132,7 +162,7 @@ module Ci def retryable? builds.latest.any? do |build| - build.failed? && build.retryable? + (build.failed? || build.canceled?) && build.retryable? end end @@ -185,7 +215,7 @@ module Ci end def has_warnings? - builds.latest.ignored.any? + builds.latest.failed_but_allowed.any? end def config_processor @@ -240,14 +270,16 @@ module Ci Ci::ProcessPipelineService.new(project, user).execute(self) end - def build_updated - case latest_builds_status - when 'pending' then enqueue - when 'running' then run - when 'success' then succeed - when 'failed' then drop - when 'canceled' then cancel - when 'skipped' then skip + def update_status + Gitlab::OptimisticLocking.retry_lock(self) do + case latest_builds_status + when 'pending' then enqueue + when 'running' then run + when 'success' then succeed + when 'failed' then drop + when 'canceled' then cancel + when 'skipped' then skip + end end end @@ -276,6 +308,14 @@ module Ci project.execute_services(data, :pipeline_hooks) end + # Merge requests for which the current pipeline is running against + # the merge request's latest commit. + def merge_requests + @merge_requests ||= project.merge_requests + .where(source_branch: self.ref) + .select { |merge_request| merge_request.pipeline.try(:id) == self.id } + end + private def pipeline_data diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ed5d4b13b7e..123930273e0 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -2,13 +2,13 @@ module Ci class Runner < ActiveRecord::Base extend Ci::Model - LAST_CONTACT_TIME = 2.hours.ago + LAST_CONTACT_TIME = 1.hour.ago AVAILABLE_SCOPES = %w[specific shared active paused online] FORM_EDITABLE = %i[description tag_list active run_untagged locked] - has_many :builds, class_name: 'Ci::Build' - has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject' - has_many :projects, through: :runner_projects, class_name: '::Project', foreign_key: :gl_project_id + has_many :builds + has_many :runner_projects, dependent: :destroy + has_many :projects, through: :runner_projects, foreign_key: :gl_project_id has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 4b44ffa886e..1f9baeca5b1 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -2,8 +2,8 @@ module Ci class RunnerProject < ActiveRecord::Base extend Ci::Model - belongs_to :runner, class_name: 'Ci::Runner' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :runner + belongs_to :project, foreign_key: :gl_project_id validates_uniqueness_of :runner_id, scope: :gl_project_id end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index a0b19b51a12..62889fe80d8 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -4,8 +4,8 @@ module Ci acts_as_paranoid - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id - has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest' + belongs_to :project, foreign_key: :gl_project_id + has_many :trigger_requests, dependent: :destroy validates_presence_of :token validates_uniqueness_of :token diff --git a/app/models/ci/trigger_request.rb b/app/models/ci/trigger_request.rb index fc674871743..2b807731d0d 100644 --- a/app/models/ci/trigger_request.rb +++ b/app/models/ci/trigger_request.rb @@ -2,9 +2,9 @@ module Ci class TriggerRequest < ActiveRecord::Base extend Ci::Model - belongs_to :trigger, class_name: 'Ci::Trigger' - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id - has_many :builds, class_name: 'Ci::Build' + belongs_to :trigger + belongs_to :pipeline, foreign_key: :commit_id + has_many :builds serialize :variables diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 6959223aed9..94d9e2b3208 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -2,7 +2,7 @@ module Ci class Variable < ActiveRecord::Base extend Ci::Model - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :project, foreign_key: :gl_project_id validates_uniqueness_of :key, scope: :gl_project_id validates :key, diff --git a/app/models/commit.rb b/app/models/commit.rb index e64fd1e0c1b..9e7fde9503d 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -226,12 +226,19 @@ class Commit end def pipelines - @pipeline ||= project.pipelines.where(sha: sha) + project.pipelines.where(sha: sha) end - def status - return @status if defined?(@status) - @status ||= pipelines.status + def status(ref = nil) + @statuses ||= {} + + if @statuses.key?(ref) + @statuses[ref] + elsif ref + @statuses[ref] = pipelines.where(ref: ref).status + else + @statuses[ref] = pipelines.status + end end def revert_branch_name diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 656a242c265..ac2477fd973 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -80,7 +80,7 @@ class CommitRange end def inspect - %(#<#{self.class}:#{object_id} #{to_s}>) + %(#<#{self.class}:#{object_id} #{self}>) end def to_s diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index c85561291c8..c345bf293c9 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -1,10 +1,11 @@ class CommitStatus < ActiveRecord::Base include HasStatus include Importable + include AfterCommitQueue self.table_name = 'ci_builds' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :project, foreign_key: :gl_project_id belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id belongs_to :user @@ -24,7 +25,22 @@ class CommitStatus < ActiveRecord::Base scope :retried, -> { where.not(id: latest) } scope :ordered, -> { order(:name) } - scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + quoted_when = connection.quote_column_name('when') + # We want to ignore failed_but_allowed jobs + where("allow_failure = ? OR status IN (?)", + false, all_state_names - [:failed, :canceled]). + # We want to ignore skipped manual jobs + where("#{quoted_when} <> ? OR status <> ?", 'manual', 'skipped'). + # We want to ignore skipped on_failure + where("#{quoted_when} <> ? OR status <> ?", 'on_failure', 'skipped') + end + scope :latest_ci_stages, -> { latest.ordered.includes(project: :namespace) } scope :retried_ci_stages, -> { retried.ordered.includes(project: :namespace) } @@ -57,33 +73,37 @@ class CommitStatus < ActiveRecord::Base transition [:created, :pending, :running] => :canceled end - after_transition created: [:pending, :running] do |commit_status| - commit_status.update_attributes queued_at: Time.now + before_transition created: [:pending, :running] do |commit_status| + commit_status.queued_at = Time.now end - after_transition [:created, :pending] => :running do |commit_status| - commit_status.update_attributes started_at: Time.now + before_transition [:created, :pending] => :running do |commit_status| + commit_status.started_at = Time.now end - after_transition any => [:success, :failed, :canceled] do |commit_status| - commit_status.update_attributes finished_at: Time.now + before_transition any => [:success, :failed, :canceled] do |commit_status| + commit_status.finished_at = Time.now end after_transition do |commit_status, transition| - commit_status.pipeline.try(:build_updated) unless transition.loopback? - end - - after_transition any => [:success, :failed, :canceled] do |commit_status| - commit_status.pipeline.try(:process!) - true - end - - after_transition [:created, :pending, :running] => :success do |commit_status| - MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.pipeline.project, nil).trigger(commit_status) + next if transition.loopback? + + commit_status.run_after_commit do + pipeline.try do |pipeline| + if complete? + PipelineProcessWorker.perform_async(pipeline.id) + else + PipelineUpdateWorker.perform_async(pipeline.id) + end + end + end end after_transition any => :failed do |commit_status| - MergeRequests::AddTodoWhenBuildFailsService.new(commit_status.pipeline.project, nil).execute(commit_status) + commit_status.run_after_commit do + MergeRequests::AddTodoWhenBuildFailsService + .new(pipeline.project, nil).execute(self) + end end end @@ -111,19 +131,23 @@ class CommitStatus < ActiveRecord::Base end end - def ignored? + def failed_but_allowed? allow_failure? && (failed? || canceled?) end + def duration + calculate_duration + end + def playable? false end - def duration - calculate_duration + def stuck? + false end - def stuck? + def has_trace? false end end diff --git a/app/models/compare.rb b/app/models/compare.rb index 4856510f526..3a8bbcb1acd 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -11,9 +11,10 @@ class Compare end end - def initialize(compare, project) + def initialize(compare, project, straight: false) @compare = compare @project = project + @straight = straight end def commits @@ -45,6 +46,18 @@ class Compare end end + def start_commit_sha + start_commit.try(:sha) + end + + def base_commit_sha + base_commit.try(:sha) + end + + def head_commit_sha + commit.try(:sha) + end + def raw_diffs(*args) @compare.diffs(*args) end @@ -58,9 +71,9 @@ class Compare def diff_refs Gitlab::Diff::DiffRefs.new( - base_sha: base_commit.try(:sha), - start_sha: start_commit.try(:sha), - head_sha: commit.try(:sha) + base_sha: @straight ? start_commit_sha : base_commit_sha, + start_sha: start_commit_sha, + head_sha: head_commit_sha ) end end diff --git a/app/models/concerns/access_requestable.rb b/app/models/concerns/access_requestable.rb index eedd32a729f..62bc6b809f4 100644 --- a/app/models/concerns/access_requestable.rb +++ b/app/models/concerns/access_requestable.rb @@ -8,9 +8,6 @@ module AccessRequestable extend ActiveSupport::Concern def request_access(user) - members.create( - access_level: Gitlab::Access::DEVELOPER, - user: user, - requested_at: Time.now.utc) + Members::RequestAccessService.new(self, user).execute end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index d8d4575bb4d..073ac4c1b65 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -71,6 +71,12 @@ module Awardable end end + def user_authored?(current_user) + author = self.respond_to?(:author) ? self.author : self.user + + author == current_user + end + def awarded_emoji?(emoji_name, current_user) award_emoji.where(name: emoji_name, user: current_user).exists? end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb new file mode 100644 index 00000000000..90bd6490a02 --- /dev/null +++ b/app/models/concerns/cache_markdown_field.rb @@ -0,0 +1,131 @@ +# This module takes care of updating cache columns for Markdown-containing +# fields. Use like this in the body of your class: +# +# include CacheMarkdownField +# cache_markdown_field :foo +# cache_markdown_field :bar +# cache_markdown_field :baz, pipeline: :single_line +# +# Corresponding foo_html, bar_html and baz_html fields should exist. +module CacheMarkdownField + # Knows about the relationship between markdown and html field names, and + # stores the rendering contexts for the latter + class FieldData + extend Forwardable + + def initialize + @data = {} + end + + def_delegators :@data, :[], :[]= + def_delegator :@data, :keys, :markdown_fields + + def html_field(markdown_field) + "#{markdown_field}_html" + end + + def html_fields + markdown_fields.map {|field| html_field(field) } + end + end + + # Dynamic registries don't really work in Rails as it's not guaranteed that + # every class will be loaded, so hardcode the list. + CACHING_CLASSES = %w[ + AbuseReport + Appearance + ApplicationSetting + BroadcastMessage + Issue + Label + MergeRequest + Milestone + Namespace + Note + Project + Release + Snippet + ] + + def self.caching_classes + CACHING_CLASSES.map(&:constantize) + end + + extend ActiveSupport::Concern + + included do + cattr_reader :cached_markdown_fields do + FieldData.new + end + + # Returns the default Banzai render context for the cached markdown field. + def banzai_render_context(field) + raise ArgumentError.new("Unknown field: #{field.inspect}") unless + cached_markdown_fields.markdown_fields.include?(field) + + # Always include a project key, or Banzai complains + project = self.project if self.respond_to?(:project) + context = cached_markdown_fields[field].merge(project: project) + + # Banzai is less strict about authors, so don't always have an author key + context[:author] = self.author if self.respond_to?(:author) + + context + end + + # Allow callers to look up the cache field name, rather than hardcoding it + def markdown_cache_field_for(field) + raise ArgumentError.new("Unknown field: #{field}") unless + cached_markdown_fields.markdown_fields.include?(field) + + cached_markdown_fields.html_field(field) + end + + # Always exclude _html fields from attributes (including serialization). + # They contain unredacted HTML, which would be a security issue + alias_method :attributes_before_markdown_cache, :attributes + def attributes + attrs = attributes_before_markdown_cache + + cached_markdown_fields.html_fields.each do |field| + attrs.delete(field) + end + + attrs + end + end + + class_methods do + private + + # Specify that a field is markdown. Its rendered output will be cached in + # a corresponding _html field. Any custom rendering options may be provided + # as a context. + def cache_markdown_field(markdown_field, context = {}) + raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless + CacheMarkdownField::CACHING_CLASSES.include?(self.to_s) + + cached_markdown_fields[markdown_field] = context + + html_field = cached_markdown_fields.html_field(markdown_field) + cache_method = "#{markdown_field}_cache_refresh".to_sym + invalidation_method = "#{html_field}_invalidated?".to_sym + + define_method(cache_method) do + html = Banzai::Renderer.cacheless_render_field(self, markdown_field) + __send__("#{html_field}=", html) + true + end + + # The HTML becomes invalid if any dependent fields change. For now, assume + # author and project invalidate the cache in all circumstances. + define_method(invalidation_method) do + changed_fields = changed_attributes.keys + invalidations = changed_fields & [markdown_field.to_s, "author", "project"] + !invalidations.empty? + end + + before_save cache_method, if: invalidation_method + end + end +end diff --git a/app/models/concerns/expirable.rb b/app/models/concerns/expirable.rb index be93435453b..b66ba08dc59 100644 --- a/app/models/concerns/expirable.rb +++ b/app/models/concerns/expirable.rb @@ -5,11 +5,15 @@ module Expirable scope :expired, -> { where('expires_at <= ?', Time.current) } end + def expired? + expires? && expires_at <= Time.current + end + def expires? expires_at.present? end def expires_soon? - expires_at < 7.days.from_now + expires? && expires_at < 7.days.from_now end end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index d658552f695..ef3e73a4072 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -5,35 +5,36 @@ module HasStatus STARTED_STATUSES = %w[running success failed skipped] ACTIVE_STATUSES = %w[pending running] COMPLETED_STATUSES = %w[success failed canceled] + ORDERED_STATUSES = %w[failed pending running canceled success skipped] class_methods do def status_sql - scope = all + scope = if respond_to?(:exclude_ignored) + exclude_ignored + else + all + end builds = scope.select('count(*)').to_sql created = scope.created.select('count(*)').to_sql success = scope.success.select('count(*)').to_sql - ignored = scope.ignored.select('count(*)').to_sql if scope.respond_to?(:ignored) - ignored ||= '0' pending = scope.pending.select('count(*)').to_sql running = scope.running.select('count(*)').to_sql - canceled = scope.canceled.select('count(*)').to_sql skipped = scope.skipped.select('count(*)').to_sql + canceled = scope.canceled.select('count(*)').to_sql - deduce_status = "(CASE - WHEN (#{builds})=(#{created}) THEN NULL - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success})+(#{ignored})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{created})+(#{pending})+(#{skipped}) THEN 'pending' - WHEN (#{builds})=(#{canceled})+(#{success})+(#{ignored})+(#{skipped}) THEN 'canceled' + "(CASE + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' WHEN (#{running})+(#{pending})+(#{created})>0 THEN 'running' ELSE 'failed' END)" - - deduce_status end def status - all.pluck(self.status_sql).first + all.pluck(status_sql).first end def started_at @@ -43,6 +44,10 @@ module HasStatus def finished_at all.maximum(:finished_at) end + + def all_state_names + state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } + end end included do diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 22231b2e0f0..69d8afc45da 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -6,13 +6,18 @@ # module Issuable extend ActiveSupport::Concern + include CacheMarkdownField include Participable include Mentionable include Subscribable include StripAttribute include Awardable + include Taskable included do + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description + belongs_to :author, class_name: "User" belongs_to :assignee, class_name: "User" belongs_to :updated_by, class_name: "User" @@ -28,10 +33,13 @@ module Issuable loaded? && to_a.all? { |note| note.association(:award_emoji).loaded? } end end + has_many :label_links, as: :target, dependent: :destroy has_many :labels, through: :label_links has_many :todos, as: :target, dependent: :destroy + has_one :metrics + validates :author, presence: true validates :title, presence: true, length: { within: 0..255 } @@ -81,6 +89,7 @@ module Issuable acts_as_paranoid after_save :update_assignee_cache_counts, if: :assignee_id_changed? + after_save :record_metrics def update_assignee_cache_counts # make sure we flush the cache for both the old *and* new assignee @@ -137,8 +146,14 @@ module Issuable end def order_labels_priority(excluded_labels: []) - condition_field = "#{table_name}.id" - highest_priority = highest_label_priority(name, condition_field, excluded_labels: excluded_labels).to_sql + params = { + target_type: name, + target_column: "#{table_name}.id", + project_column: "#{table_name}.#{project_foreign_key}", + excluded_labels: excluded_labels + } + + highest_priority = highest_label_priority(params).to_sql select("#{table_name}.*, (#{highest_priority}) AS highest_priority"). group(arel_table[:id]). @@ -168,6 +183,10 @@ module Issuable grouping_columns end + + def to_ability_name + model_name.singular + end end def today? @@ -196,11 +215,7 @@ module Issuable end end - def user_authored?(user) - user == author - end - - def subscribed_without_subscriptions?(user) + def subscribed_without_subscriptions?(user, project) participants(user).include?(user) end @@ -226,18 +241,6 @@ module Issuable labels.order('title ASC').pluck(:title) end - def remove_labels - labels.delete_all - end - - def add_labels_by_names(label_names) - label_names.each do |label_name| - label = project.labels.create_with(color: Label::DEFAULT_COLOR). - find_or_create_by(title: label_name.strip) - self.labels << label - end - end - # Convert this Issuable class name to a format usable by Ability definitions # # Examples: @@ -245,7 +248,18 @@ module Issuable # issuable.class # => MergeRequest # issuable.to_ability_name # => "merge_request" def to_ability_name - self.class.to_s.underscore + self.class.to_ability_name + end + + # Convert this Issuable class name to a format usable by notifications. + # + # Examples: + # + # issuable.class # => MergeRequest + # issuable.human_class_name # => "merge request" + + def human_class_name + @human_class_name ||= self.class.name.titleize.downcase end # Returns a Hash of attributes to be used for Twitter card metadata @@ -286,4 +300,14 @@ module Issuable def can_move?(*) false end + + def assignee_or_author?(user) + # We're comparing IDs here so we don't need to load any associations. + author_id == user.id || assignee_id == user.id + end + + def record_metrics + metrics = self.metrics || create_metrics + metrics.record! + end end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index ec9e0f1b1d0..8ab0401d288 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -1,6 +1,6 @@ # == Mentionable concern # -# Contains functionality related to objects that can mention Users, Issues, MergeRequests, or Commits by +# Contains functionality related to objects that can mention Users, Issues, MergeRequests, Commits or Snippets by # GFM references. # # Used by Issue, Note, MergeRequest, and Commit. @@ -43,19 +43,15 @@ module Mentionable self end - def all_references(current_user = nil, text = nil, extractor: nil) + def all_references(current_user = nil, extractor: nil) extractor ||= Gitlab::ReferenceExtractor. new(project, current_user) - if text - extractor.analyze(text, author: author) - else - self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) - options = options.merge(cache_key: [self, attr], author: author) + self.class.mentionable_attrs.each do |attr, options| + text = __send__(attr) + options = options.merge(cache_key: [self, attr], author: author) - extractor.analyze(text, options) - end + extractor.analyze(text, options) end extractor @@ -66,8 +62,8 @@ module Mentionable end # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. - def referenced_mentionables(current_user = self.author, text = nil) - refs = all_references(current_user, text) + def referenced_mentionables(current_user = self.author) + refs = all_references(current_user) refs = (refs.issues + refs.merge_requests + refs.commits) # We're using this method instead of Array diffing because that requires @@ -77,8 +73,8 @@ module Mentionable end # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+. - def create_cross_references!(author = self.author, without = [], text = nil) - refs = referenced_mentionables(author, text) + def create_cross_references!(author = self.author, without = []) + refs = referenced_mentionables(author) # We're using this method instead of Array diffing because that requires # both of the object's `hash` values to be the same, which may not be the @@ -97,10 +93,7 @@ module Mentionable return if changes.empty? - original_text = changes.collect { |_, vals| vals.first }.join(' ') - - preexisting = referenced_mentionables(author, original_text) - create_cross_references!(author, preexisting) + create_cross_references!(author) end private diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 7bcc78247ba..e65fc9eaa09 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -23,7 +23,31 @@ module Milestoneish (due_date - Date.today).to_i end + def elapsed_days + return 0 if !start_date || start_date.future? + + (Date.today - start_date).to_i + end + def issues_visible_to_user(user = nil) issues.visible_to_user(user) end + + def upcoming? + start_date && start_date.future? + end + + def expires_at + if due_date + if due_date.past? + "expired on #{due_date.to_s(:medium)}" + else + "expires on #{due_date.to_s(:medium)}" + end + end + end + + def expired? + due_date && due_date.past? + end end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 9216122923e..6d88951c713 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -31,7 +31,7 @@ module ProjectFeaturesCompatibility def write_feature_attribute(field, value) build_project_feature unless project_feature - access_level = value == "true" ? ProjectFeature::ENABLED : ProjectFeature::DISABLED + access_level = Gitlab::Utils.to_boolean(value) ? ProjectFeature::ENABLED : ProjectFeature::DISABLED project_feature.update_attribute(field, access_level) end end diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index 5a7b36070e7..7fd0905ee81 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -1,6 +1,11 @@ module ProtectedBranchAccess extend ActiveSupport::Concern + included do + scope :master, -> { where(access_level: Gitlab::Access::MASTER) } + scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } + end + def humanize self.class.human_access_levels[self.access_level] end diff --git a/app/models/concerns/select_for_project_authorization.rb b/app/models/concerns/select_for_project_authorization.rb new file mode 100644 index 00000000000..50a1d7fc3e1 --- /dev/null +++ b/app/models/concerns/select_for_project_authorization.rb @@ -0,0 +1,9 @@ +module SelectForProjectAuthorization + extend ActiveSupport::Concern + + module ClassMethods + def select_for_project_authorization + select("members.user_id, projects.id AS project_id, members.access_level") + end + end +end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index 1ebecd86af9..7edb0acd56c 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -38,14 +38,21 @@ module Sortable private - def highest_label_priority(object_types, condition_field, excluded_labels: []) - query = Label.select(Label.arel_table[:priority].minimum). + def highest_label_priority(target_type_column: nil, target_type: nil, target_column:, project_column:, excluded_labels: []) + query = Label.select(LabelPriority.arel_table[:priority].minimum). + left_join_priorities. joins(:label_links). - where(label_links: { target_type: object_types }). - where("label_links.target_id = #{condition_field}"). + where("label_priorities.project_id = #{project_column}"). + where("label_links.target_id = #{target_column}"). reorder(nil) - query.where.not(title: excluded_labels) if excluded_labels.present? + if target_type_column + query = query.where("label_links.target_type = #{target_type_column}") + else + query = query.where(label_links: { target_type: target_type }) + end + + query = query.where.not(title: excluded_labels) if excluded_labels.present? query end diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 083257f1005..83daa9b1a64 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -12,39 +12,71 @@ module Subscribable has_many :subscriptions, dependent: :destroy, as: :subscribable end - def subscribed?(user) - if subscription = subscriptions.find_by_user_id(user.id) + def subscribed?(user, project = nil) + if subscription = subscriptions.find_by(user: user, project: project) subscription.subscribed else - subscribed_without_subscriptions?(user) + subscribed_without_subscriptions?(user, project) end end # Override this method to define custom logic to consider a subscribable as # subscribed without an explicit subscription record. - def subscribed_without_subscriptions?(user) + def subscribed_without_subscriptions?(user, project) false end - def subscribers - subscriptions.where(subscribed: true).map(&:user) + def subscribers(project) + subscriptions_available(project). + where(subscribed: true). + map(&:user) end - def toggle_subscription(user) - subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: !subscribed?(user)) + def toggle_subscription(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project). + update(subscribed: !subscribed?(user, project)) + end + + def subscribe(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project) + .update(subscribed: true) + end + + def unsubscribe(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project) + .update(subscribed: false) end - def subscribe(user) + private + + def unsubscribe_from_other_levels(user, project) + other_subscriptions = subscriptions.where(user: user) + + other_subscriptions = + if project.blank? + other_subscriptions.where.not(project: nil) + else + other_subscriptions.where(project: nil) + end + + other_subscriptions.update_all(subscribed: false) + end + + def find_or_initialize_subscription(user, project) subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: true) + find_or_initialize_by(user_id: user.id, project_id: project.try(:id)) end - def unsubscribe(user) + def subscriptions_available(project) + t = Subscription.arel_table + subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: false) + where(t[:project_id].eq(nil).or(t[:project_id].eq(project.try(:id)))) end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index a3ac577cf3e..ebc75100a54 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -53,10 +53,22 @@ module Taskable # Return a string that describes the current state of this Taskable's task # list items, e.g. "12 of 20 tasks completed" - def task_status + def task_status(short: false) return '' if description.blank? + prep, completed = if short + ['/', ''] + else + [' of ', ' completed'] + end + sum = tasks.summary - "#{sum.complete_count} of #{sum.item_count} #{'task'.pluralize(sum.item_count)} completed" + "#{sum.complete_count}#{prep}#{sum.item_count} #{'task'.pluralize(sum.item_count)}#{completed}" + end + + # Return a short string that describes the current state of this Taskable's + # task list items -- for small screens + def task_status_short + task_status(short: true) end end diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 24c7b26d223..04d30f46210 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -4,17 +4,21 @@ module TokenAuthenticatable private def write_new_token(token_field) - new_token = generate_token(token_field) + new_token = generate_available_token(token_field) write_attribute(token_field, new_token) end - def generate_token(token_field) + def generate_available_token(token_field) loop do - token = Devise.friendly_token + token = generate_token(token_field) break token unless self.class.unscoped.find_by(token_field => token) end end + def generate_token(token_field) + Devise.friendly_token + end + class_methods do def authentication_token_fields @token_fields || [] diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb new file mode 100644 index 00000000000..cb8e088d21d --- /dev/null +++ b/app/models/cycle_analytics.rb @@ -0,0 +1,61 @@ +class CycleAnalytics + STAGES = %i[issue plan code test review staging production].freeze + + def initialize(project, from:) + @project = project + @from = from + @fetcher = Gitlab::CycleAnalytics::MetricsFetcher.new(project: project, from: from, branch: nil) + end + + def summary + @summary ||= Summary.new(@project, from: @from) + end + + def permissions(user:) + Gitlab::CycleAnalytics::Permissions.get(user: user, project: @project) + end + + def issue + @fetcher.calculate_metric(:issue, + Issue.arel_table[:created_at], + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]]) + end + + def plan + @fetcher.calculate_metric(:plan, + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]], + Issue::Metrics.arel_table[:first_mentioned_in_commit_at]) + end + + def code + @fetcher.calculate_metric(:code, + Issue::Metrics.arel_table[:first_mentioned_in_commit_at], + MergeRequest.arel_table[:created_at]) + end + + def test + @fetcher.calculate_metric(:test, + MergeRequest::Metrics.arel_table[:latest_build_started_at], + MergeRequest::Metrics.arel_table[:latest_build_finished_at]) + end + + def review + @fetcher.calculate_metric(:review, + MergeRequest.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:merged_at]) + end + + def staging + @fetcher.calculate_metric(:staging, + MergeRequest::Metrics.arel_table[:merged_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end + + def production + @fetcher.calculate_metric(:production, + Issue.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end +end diff --git a/app/models/cycle_analytics/summary.rb b/app/models/cycle_analytics/summary.rb new file mode 100644 index 00000000000..b46db449bf3 --- /dev/null +++ b/app/models/cycle_analytics/summary.rb @@ -0,0 +1,42 @@ +class CycleAnalytics + class Summary + def initialize(project, from:) + @project = project + @from = from + end + + def new_issues + @project.issues.created_after(@from).count + end + + def commits + ref = @project.default_branch.presence + count_commits_for(ref) + end + + def deploys + @project.deployments.where("created_at > ?", @from).count + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits_for(ref) + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + raw_output = IO.popen(cmd) { |io| io.read } + raw_output.lines.count + end + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 1e338889714..91d85c2279b 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -11,7 +11,7 @@ class Deployment < ActiveRecord::Base delegate :name, to: :environment, prefix: true - after_save :keep_around_commit + after_create :create_ref def commit project.commit(sha) @@ -29,17 +29,79 @@ class Deployment < ActiveRecord::Base self == environment.last_deployment end - def keep_around_commit - project.repository.keep_around(self.sha) + def create_ref + project.repository.create_ref(ref, ref_path) end def manual_actions - deployable.try(:other_actions) + @manual_actions ||= deployable.try(:other_actions) end def includes_commit?(commit) return false unless commit - project.repository.is_ancestor?(commit.id, sha) + # Before 8.10, deployments didn't have keep-around refs. Any deployment + # created before then could have a `sha` referring to a commit that no + # longer exists in the repository, so just ignore those. + begin + project.repository.is_ancestor?(commit.id, sha) + rescue Rugged::OdbError + false + end + end + + def update_merge_request_metrics! + return unless environment.update_merge_request_metrics? + + merge_requests = project.merge_requests. + joins(:metrics). + where(target_branch: self.ref, merge_request_metrics: { first_deployed_to_production_at: nil }). + where("merge_request_metrics.merged_at <= ?", self.created_at) + + if previous_deployment + merge_requests = merge_requests.where("merge_request_metrics.merged_at >= ?", previous_deployment.created_at) + end + + # Need to use `map` instead of `select` because MySQL doesn't allow `SELECT`ing from the same table + # that we're updating. + merge_request_ids = + if Gitlab::Database.postgresql? + merge_requests.select(:id) + elsif Gitlab::Database.mysql? + merge_requests.map(&:id) + end + + MergeRequest::Metrics. + where(merge_request_id: merge_request_ids, first_deployed_to_production_at: nil). + update_all(first_deployed_to_production_at: self.created_at) + end + + def previous_deployment + @previous_deployment ||= + project.deployments.joins(:environment). + where(environments: { name: self.environment.name }, ref: self.ref). + where.not(id: self.id). + take + end + + def stop_action + return nil unless on_stop.present? + return nil unless manual_actions + + @stop_action ||= manual_actions.find_by(name: on_stop) + end + + def stoppable? + stop_action.present? + end + + def formatted_deployment_time + created_at.to_time.in_time_zone.to_s(:medium) + end + + private + + def ref_path + File.join(environment.ref_path, 'deployments', iid.to_s) end end diff --git a/app/models/email.rb b/app/models/email.rb index 32a412ab878..826d4f16edb 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -7,10 +7,8 @@ class Email < ActiveRecord::Base validates :email, presence: true, uniqueness: true, email: true validate :unique_email, if: ->(email) { email.email_changed? } - before_validation :cleanup_email - - def cleanup_email - self.email = self.email.downcase.strip + def email=(value) + write_attribute(:email, value.downcase.strip) end def unique_email diff --git a/app/models/environment.rb b/app/models/environment.rb index 75e6f869786..a7f4156fc2e 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -4,6 +4,7 @@ class Environment < ActiveRecord::Base has_many :deployments before_validation :nullify_external_url + before_save :set_environment_type validates :name, presence: true, @@ -18,6 +19,28 @@ class Environment < ActiveRecord::Base allow_nil: true, addressable_url: true + delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true + + scope :available, -> { with_state(:available) } + scope :stopped, -> { with_state(:stopped) } + + state_machine :state, initial: :available do + event :start do + transition stopped: :available + end + + event :stop do + transition available: :stopped + end + + state :available + state :stopped + end + + def recently_updated_on_branch?(ref) + ref.to_s == last_deployment.try(:ref) + end + def last_deployment deployments.last end @@ -26,9 +49,62 @@ class Environment < ActiveRecord::Base self.external_url = nil if self.external_url.blank? end + def set_environment_type + names = name.split('/') + + self.environment_type = + if names.many? + names.first + else + nil + end + end + def includes_commit?(commit) return false unless last_deployment last_deployment.includes_commit?(commit) end + + def update_merge_request_metrics? + self.name == "production" + end + + def first_deployment_for(commit) + ref = project.repository.ref_name_for_sha(ref_path, commit.sha) + + return nil unless ref + + deployment_iid = ref.split('/').last + deployments.find_by(iid: deployment_iid) + end + + def ref_path + "refs/environments/#{Shellwords.shellescape(name)}" + end + + def formatted_external_url + return nil unless external_url + + external_url.gsub(/\A.*?:\/\//, '') + end + + def stoppable? + available? && stop_action.present? + end + + def stop!(current_user) + return unless stoppable? + + stop + stop_action.play(current_user) + end + + def actions_for(environment) + return [] unless manual_actions + + manual_actions.select do |action| + action.expanded_environment_name == environment + end + end end diff --git a/app/models/event.rb b/app/models/event.rb index a0b7b0dc2b5..21eaca917b8 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,6 +1,6 @@ class Event < ActiveRecord::Base include Sortable - default_scope { where.not(author_id: nil) } + default_scope { reorder(nil).where.not(author_id: nil) } CREATED = 1 UPDATED = 2 @@ -12,6 +12,9 @@ class Event < ActiveRecord::Base JOINED = 8 # User joined project LEFT = 9 # User left project DESTROYED = 10 + EXPIRED = 11 # User left project due to expiry + + RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour delegate :name, :email, to: :author, prefix: true, allow_nil: true delegate :title, to: :issue, prefix: true, allow_nil: true @@ -46,6 +49,7 @@ class Event < ActiveRecord::Base update_all(updated_at: Time.now) end + # Update Gitlab::ContributionsCalendar#activity_dates if this changes def contributions where("action = ? OR (target_type in (?) AND action in (?))", Event::PUSHED, ["MergeRequest", "Issue"], @@ -58,16 +62,18 @@ class Event < ActiveRecord::Base end def visible_to_user?(user = nil) - if push? - true + if push? || commit_note? + Ability.allowed?(user, :download_code, project) elsif membership_changed? true elsif created_project? true elsif issue? || issue_note? Ability.allowed?(user, :read_issue, note? ? note_target : target) + elsif merge_request? || merge_request_note? + Ability.allowed?(user, :read_merge_request, note? ? note_target : target) else - ((merge_request? || note?) && target.present?) || milestone? + milestone? end end @@ -111,6 +117,10 @@ class Event < ActiveRecord::Base action == LEFT end + def expired? + action == EXPIRED + end + def destroyed? action == DESTROYED end @@ -120,7 +130,7 @@ class Event < ActiveRecord::Base end def membership_changed? - joined? || left? + joined? || left? || expired? end def created_project? @@ -180,6 +190,8 @@ class Event < ActiveRecord::Base 'joined' elsif left? 'left' + elsif expired? + 'removed due to membership expiration from' elsif destroyed? 'destroyed' elsif commented? @@ -271,15 +283,19 @@ class Event < ActiveRecord::Base end def commit_note? - target.for_commit? + note? && target && target.for_commit? end def issue_note? note? && target && target.for_issue? end + def merge_request_note? + note? && target && target.for_merge_request? + end + def project_snippet_note? - target.for_snippet? + note? && target && target.for_snippet? end def note_target @@ -324,8 +340,22 @@ class Event < ActiveRecord::Base end def reset_project_activity - if project && Gitlab::ExclusiveLease.new("project:update_last_activity_at:#{project.id}", timeout: 60).try_obtain - project.update_column(:last_activity_at, self.created_at) - end + return unless project + + # Don't bother updating if we know the project was updated recently. + return if recent_update? + + # At this point it's possible for multiple threads/processes to try to + # update the project. Only one query should actually perform the update, + # hence we add the extra WHERE clause for last_activity_at. + Project.unscoped.where(id: project_id). + where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago). + update_all(last_activity_at: created_at) + end + + private + + def recent_update? + project.last_activity_at > RESET_PROJECT_ACTIVITY_INTERVAL.ago end end diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index b7894c99846..91b508eb325 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -29,6 +29,10 @@ class ExternalIssue @project end + def project_id + @project.id + end + # Pattern used to extract `JIRA-123` issue references from text def self.reference_pattern @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} diff --git a/app/models/global_label.rb b/app/models/global_label.rb index ddd4bad5c21..698a7bbd327 100644 --- a/app/models/global_label.rb +++ b/app/models/global_label.rb @@ -4,6 +4,10 @@ class GlobalLabel delegate :color, :description, to: :@first_label + def for_display + @first_label + end + def self.build_collection(labels) labels = labels.group_by(&:title) diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index da7c265a371..b01607dcda9 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -4,11 +4,16 @@ class GlobalMilestone attr_accessor :title, :milestones alias_attribute :name, :title + def for_display + @first_milestone + end + def self.build_collection(milestones) milestones = milestones.group_by(&:title) milestones.map do |title, milestones| - new(title, milestones) + milestones_relation = Milestone.where(id: milestones.map(&:id)) + new(title, milestones_relation) end end @@ -16,32 +21,23 @@ class GlobalMilestone @title = title @name = title @milestones = milestones + @first_milestone = milestones.find {|m| m.description.present? } || milestones.first end def safe_title @title.to_slug.normalize.to_s end - def expired? - if due_date - due_date.past? - else - false - end - end - def projects - @projects ||= Project.for_milestones(milestones.map(&:id)) + @projects ||= Project.for_milestones(milestones.select(:id)) end def state - state = milestones.map { |milestone| milestone.state } - - if state.count('closed') == state.size - 'closed' - else - 'active' + milestones.each do |milestone| + return 'active' if milestone.state != 'closed' end + + 'closed' end def active? @@ -53,19 +49,19 @@ class GlobalMilestone end def issues - @issues ||= Issue.of_milestones(milestones.map(&:id)).includes(:project) + @issues ||= Issue.of_milestones(milestones.select(:id)).includes(:project, :assignee, :labels) end def merge_requests - @merge_requests ||= MergeRequest.of_milestones(milestones.map(&:id)).includes(:target_project) + @merge_requests ||= MergeRequest.of_milestones(milestones.select(:id)).includes(:target_project, :assignee, :labels) end def participants - @participants ||= milestones.map(&:participants).flatten.compact.uniq + @participants ||= milestones.includes(:participants).map(&:participants).flatten.compact.uniq end def labels - @labels ||= GlobalLabel.build_collection(milestones.map(&:labels).flatten) + @labels ||= GlobalLabel.build_collection(milestones.includes(:labels).map(&:labels).flatten) .sort_by!(&:title) end @@ -75,18 +71,15 @@ class GlobalMilestone @due_date = if @milestones.all? { |x| x.due_date == @milestones.first.due_date } @milestones.first.due_date - else - nil end end - def expires_at - if due_date - if due_date.past? - "expired on #{due_date.to_s(:medium)}" - else - "expires on #{due_date.to_s(:medium)}" + def start_date + return @start_date if defined?(@start_date) + + @start_date = + if @milestones.all? { |x| x.start_date == @milestones.first.start_date } + @milestones.first.start_date end - end end end diff --git a/app/models/group.rb b/app/models/group.rb index aefb94b2ada..4248e1162d8 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -5,8 +5,9 @@ class Group < Namespace include Gitlab::VisibilityLevel include AccessRequestable include Referable + include SelectForProjectAuthorization - has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' + has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :group_members has_many :users, through: :group_members has_many :owners, @@ -19,6 +20,7 @@ class Group < Namespace has_many :project_group_links, dependent: :destroy has_many :shared_projects, through: :project_group_links, source: :project has_many :notification_settings, dependent: :destroy, as: :source + has_many :labels, class_name: 'GroupLabel' validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :visibility_level_allowed_by_projects @@ -60,6 +62,16 @@ class Group < Namespace def visible_to_user(user) where(id: user.authorized_groups.select(:id).reorder(nil)) end + + def select_for_project_authorization + if current_scope.joins_values.include?(:shared_projects) + joins('INNER JOIN namespaces project_namespace ON project_namespace.id = projects.namespace_id') + .where('project_namespace.share_with_group_lock = ?', false) + .select("members.user_id, projects.id AS project_id, LEAST(project_group_links.group_access, members.access_level) AS access_level") + else + super + end + end end def to_reference(_from_project = nil) @@ -67,7 +79,7 @@ class Group < Namespace end def web_url - Gitlab::Routing.url_helpers.group_url(self) + Gitlab::Routing.url_helpers.group_canonical_url(self) end def human_name @@ -102,40 +114,44 @@ class Group < Namespace self[:lfs_enabled] end - def add_users(user_ids, access_level, current_user: nil, expires_at: nil) - user_ids.each do |user_id| - Member.add_user( - self.group_members, - user_id, - access_level, - current_user: current_user, - expires_at: expires_at - ) - end + def add_users(users, access_level, current_user: nil, expires_at: nil) + GroupMember.add_users_to_group( + self, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) end def add_user(user, access_level, current_user: nil, expires_at: nil) - add_users([user], access_level, current_user: current_user, expires_at: expires_at) + GroupMember.add_user( + self, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) end def add_guest(user, current_user = nil) - add_user(user, Gitlab::Access::GUEST, current_user: current_user) + add_user(user, :guest, current_user: current_user) end def add_reporter(user, current_user = nil) - add_user(user, Gitlab::Access::REPORTER, current_user: current_user) + add_user(user, :reporter, current_user: current_user) end def add_developer(user, current_user = nil) - add_user(user, Gitlab::Access::DEVELOPER, current_user: current_user) + add_user(user, :developer, current_user: current_user) end def add_master(user, current_user = nil) - add_user(user, Gitlab::Access::MASTER, current_user: current_user) + add_user(user, :master, current_user: current_user) end def add_owner(user, current_user = nil) - add_user(user, Gitlab::Access::OWNER, current_user: current_user) + add_user(user, :owner, current_user: current_user) end def has_owner?(user) @@ -171,4 +187,8 @@ class Group < Namespace def system_hook_service SystemHooksService.new end + + def refresh_members_authorized_projects + UserProjectAccessChangedService.new(users.pluck(:id)).execute + end end diff --git a/app/models/group_label.rb b/app/models/group_label.rb new file mode 100644 index 00000000000..68841ace2e6 --- /dev/null +++ b/app/models/group_label.rb @@ -0,0 +1,15 @@ +class GroupLabel < Label + belongs_to :group + + validates :group, presence: true + + alias_attribute :subject, :group + + def subject_foreign_key + 'group_id' + end + + def to_reference(source_project = nil, target_project = nil, format: :id) + super(source_project, target_project, format: format) + end +end diff --git a/app/models/guest.rb b/app/models/guest.rb new file mode 100644 index 00000000000..01285ca1264 --- /dev/null +++ b/app/models/guest.rb @@ -0,0 +1,7 @@ +class Guest + class << self + def can?(action, subject) + Ability.allowed?(nil, action, subject) + end + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 788611305fe..dd0cb75f9a8 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -5,7 +5,6 @@ class Issue < ActiveRecord::Base include Issuable include Referable include Sortable - include Taskable include Spammable include FasterCacheKeys @@ -23,6 +22,8 @@ class Issue < ActiveRecord::Base has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + validates :project, presence: true scope :cared, ->(user) { where(assignee_id: user) } @@ -36,6 +37,8 @@ class Issue < ActiveRecord::Base scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } + scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } + attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -90,7 +93,7 @@ class Issue < ActiveRecord::Base # Check if we are scoped to a specific project's issues if owner_project - if owner_project.authorized_for_user?(user, Gitlab::Access::REPORTER) + if owner_project.team.member?(user, Gitlab::Access::REPORTER) # If the project is authorized for the user, they can see all issues in the project return all else @@ -134,6 +137,10 @@ class Issue < ActiveRecord::Base reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end + def self.project_foreign_key + 'project_id' + end + def self.sort(method, excluded_labels: []) case method.to_s when 'due_date_asc' then order_due_date_asc @@ -203,7 +210,13 @@ class Issue < ActiveRecord::Base note.all_references(current_user, extractor: ext) end - ext.merge_requests.select { |mr| mr.open? && mr.closes_issue?(self) } + merge_requests = ext.merge_requests.select(&:open?) + if merge_requests.any? + ids = MergeRequestsClosingIssues.where(merge_request_id: merge_requests.map(&:id), issue_id: id).pluck(:merge_request_id) + merge_requests.select { |mr| mr.id.in?(ids) } + else + [] + end end def moved? @@ -237,10 +250,41 @@ class Issue < ActiveRecord::Base # Returns `true` if the current issue can be viewed by either a logged in User # or an anonymous user. def visible_to_user?(user = nil) + return false unless project.feature_available?(:issues, user) + user ? readable_by?(user) : publicly_visible? end + def overdue? + due_date.try(:past?) || false + end + + # Only issues on public projects should be checked for spam + def check_for_spam? + project.public? + end + + def as_json(options = {}) + super(options).tap do |json| + json[:subscribed] = subscribed?(options[:user], project) if options.has_key?(:user) && options[:user] + + if options.has_key?(:labels) + json[:labels] = labels.as_json( + project: project, + only: [:id, :title, :description, :color, :priority], + methods: [:text_color] + ) + end + end + end + + private + # Returns `true` if the given User can read the current Issue. + # + # This method duplicates the same check of issue_policy.rb + # for performance reasons, check commit: 002ad215818450d2cbbc5fa065850a953dc7ada8 + # Make sure to sync this method with issue_policy.rb def readable_by?(user) if user.admin? true @@ -261,13 +305,4 @@ class Issue < ActiveRecord::Base def publicly_visible? project.public? && !confidential? end - - def overdue? - due_date.try(:past?) || false - end - - # Only issues on public projects should be checked for spam - def check_for_spam? - project.public? - end end diff --git a/app/models/issue/metrics.rb b/app/models/issue/metrics.rb new file mode 100644 index 00000000000..012d545c440 --- /dev/null +++ b/app/models/issue/metrics.rb @@ -0,0 +1,21 @@ +class Issue::Metrics < ActiveRecord::Base + belongs_to :issue + + def record! + if issue.milestone_id.present? && self.first_associated_with_milestone_at.blank? + self.first_associated_with_milestone_at = Time.now + end + + if issue_assigned_to_list_label? && self.first_added_to_board_at.blank? + self.first_added_to_board_at = Time.now + end + + self.save + end + + private + + def issue_assigned_to_list_label? + issue.labels.any? { |label| label.lists.present? } + end +end diff --git a/app/models/issue_collection.rb b/app/models/issue_collection.rb new file mode 100644 index 00000000000..f0b7d9914c8 --- /dev/null +++ b/app/models/issue_collection.rb @@ -0,0 +1,42 @@ +# IssueCollection can be used to reduce a list of issues down to a subset. +# +# IssueCollection is not meant to be some sort of Enumerable, instead it's meant +# to take a list of issues and return a new list of issues based on some +# criteria. For example, given a list of issues you may want to return a list of +# issues that can be read or updated by a given user. +class IssueCollection + attr_reader :collection + + def initialize(collection) + @collection = collection + end + + # Returns all the issues that can be updated by the user. + def updatable_by_user(user) + return collection if user.admin? + + # Given all the issue projects we get a list of projects that the current + # user has at least reporter access to. + projects_with_reporter_access = user. + projects_with_reporter_access_limited_to(project_ids). + pluck(:id) + + collection.select do |issue| + if projects_with_reporter_access.include?(issue.project_id) + true + elsif issue.is_a?(Issue) + issue.assignee_or_author?(user) + else + false + end + end + end + + alias_method :visible_to, :updatable_by_user + + private + + def project_ids + @project_ids ||= collection.map(&:project_id).uniq + end +end diff --git a/app/models/key.rb b/app/models/key.rb index 568a60b8af3..ff8dda2dc89 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -6,7 +6,7 @@ class Key < ActiveRecord::Base belongs_to :user - before_validation :strip_white_space, :generate_fingerprint + before_validation :generate_fingerprint validates :title, presence: true, length: { within: 0..255 } validates :key, presence: true, length: { within: 0..5000 }, format: { with: /\A(ssh|ecdsa)-.*\Z/ } @@ -21,8 +21,9 @@ class Key < ActiveRecord::Base after_destroy :remove_from_shell after_destroy :post_destroy_hook - def strip_white_space - self.key = key.strip unless key.blank? + def key=(value) + value.strip! unless value.blank? + write_attribute(:key, value) end def publishable_key diff --git a/app/models/label.rb b/app/models/label.rb index a23140b7d64..d9287f2dc29 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -1,4 +1,5 @@ class Label < ActiveRecord::Base + include CacheMarkdownField include Referable include Subscribable @@ -8,38 +9,55 @@ class Label < ActiveRecord::Base None = LabelStruct.new('No Label', 'No Label') Any = LabelStruct.new('Any Label', '') + cache_markdown_field :description, pipeline: :single_line + DEFAULT_COLOR = '#428BCA' default_value_for :color, DEFAULT_COLOR - belongs_to :project - has_many :lists, dependent: :destroy + has_many :priorities, class_name: 'LabelPriority' has_many :label_links, dependent: :destroy has_many :issues, through: :label_links, source: :target, source_type: 'Issue' has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' validates :color, color: true, allow_blank: false - validates :project, presence: true, unless: Proc.new { |service| service.template? } # Don't allow ',' for label titles - validates :title, - presence: true, - format: { with: /\A[^,]+\z/ }, - uniqueness: { scope: :project_id } - - before_save :nullify_priority + validates :title, presence: true, format: { with: /\A[^,]+\z/ } + validates :title, uniqueness: { scope: [:group_id, :project_id] } default_scope { order(title: :asc) } - scope :templates, -> { where(template: true) } + scope :templates, -> { where(template: true) } + scope :with_title, ->(title) { where(title: title) } + + def self.prioritized(project) + joins(:priorities) + .where(label_priorities: { project_id: project }) + .reorder('label_priorities.priority ASC, labels.title ASC') + end + + def self.unprioritized(project) + labels = Label.arel_table + priorities = LabelPriority.arel_table + + label_priorities = labels.join(priorities, Arel::Nodes::OuterJoin). + on(labels[:id].eq(priorities[:label_id]).and(priorities[:project_id].eq(project.id))). + join_sources - def self.prioritized - where.not(priority: nil).reorder(:priority, :title) + joins(label_priorities).where(priorities[:priority].eq(nil)) end - def self.unprioritized - where(priority: nil) + def self.left_join_priorities + labels = Label.arel_table + priorities = LabelPriority.arel_table + + label_priorities = labels.join(priorities, Arel::Nodes::OuterJoin). + on(labels[:id].eq(priorities[:label_id])). + join_sources + + joins(label_priorities) end alias_attribute :name, :title @@ -74,6 +92,51 @@ class Label < ActiveRecord::Base nil end + def open_issues_count(user = nil) + issues_count(user, state: 'opened') + end + + def closed_issues_count(user = nil) + issues_count(user, state: 'closed') + end + + def open_merge_requests_count(user = nil) + params = { + subject_foreign_key => subject.id, + label_name: title, + scope: 'all', + state: 'opened' + } + + MergeRequestsFinder.new(user, params.with_indifferent_access).execute.count + end + + def prioritize!(project, value) + label_priority = priorities.find_or_initialize_by(project_id: project.id) + label_priority.priority = value + label_priority.save! + end + + def unprioritize!(project) + priorities.where(project: project).delete_all + end + + def priority(project) + priorities.find_by(project: project).try(:priority) + end + + def template? + template + end + + def text_color + LabelsHelper.text_color_for_bg(self.color) + end + + def title=(value) + write_attribute(:title, sanitize_title(value)) if value.present? + end + ## # Returns the String necessary to reference this Label in Markdown # @@ -81,49 +144,40 @@ class Label < ActiveRecord::Base # # Examples: # - # Label.first.to_reference # => "~1" - # Label.first.to_reference(format: :name) # => "~\"bug\"" - # Label.first.to_reference(project) # => "gitlab-org/gitlab-ce~1" + # Label.first.to_reference # => "~1" + # Label.first.to_reference(format: :name) # => "~\"bug\"" + # Label.first.to_reference(project1, project2) # => "gitlab-org/gitlab-ce~1" # # Returns a String # - def to_reference(from_project = nil, format: :id) + def to_reference(source_project = nil, target_project = nil, format: :id) format_reference = label_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if cross_project_reference?(from_project) - project.to_reference + reference + if cross_project_reference?(source_project, target_project) + source_project.to_reference + reference else reference end end - def open_issues_count(user = nil) - issues.visible_to_user(user).opened.count - end - - def closed_issues_count(user = nil) - issues.visible_to_user(user).closed.count - end - - def open_merge_requests_count - merge_requests.opened.count + def as_json(options = {}) + super(options).tap do |json| + json[:priority] = priority(options[:project]) if options.has_key?(:project) + end end - def template? - template - end + private - def text_color - LabelsHelper::text_color_for_bg(self.color) + def cross_project_reference?(source_project, target_project) + source_project && target_project && source_project != target_project end - def title=(value) - write_attribute(:title, sanitize_title(value)) if value.present? + def issues_count(user, params = {}) + params.merge!(subject_foreign_key => subject.id, label_name: title, scope: 'all') + IssuesFinder.new(user, params.with_indifferent_access).execute.count end - private - def label_format_reference(format = :id) raise StandardError, 'Unknown format' unless [:id, :name].include?(format) @@ -134,10 +188,6 @@ class Label < ActiveRecord::Base end end - def nullify_priority - self.priority = nil if priority.blank? - end - def sanitize_title(value) CGI.unescapeHTML(Sanitize.clean(value.to_s)) end diff --git a/app/models/label_priority.rb b/app/models/label_priority.rb new file mode 100644 index 00000000000..5b85e0b6533 --- /dev/null +++ b/app/models/label_priority.rb @@ -0,0 +1,8 @@ +class LabelPriority < ActiveRecord::Base + belongs_to :project + belongs_to :label + + validates :project, :label, :priority, presence: true + validates :label_id, uniqueness: { scope: :project_id } + validates :priority, numericality: { only_integer: true, greater_than_or_equal_to: 0 } +end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 18657c3e1c8..7712d5783e0 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -17,4 +17,10 @@ class LfsObject < ActiveRecord::Base def project_allowed_access?(project) projects.exists?(storage_project(project).id) end + + def self.destroy_unreferenced + joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") + .where(lfs_objects_projects: { id: nil }) + .destroy_all + end end diff --git a/app/models/list.rb b/app/models/list.rb index eb87decdbc8..065d75bd1dc 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -26,6 +26,17 @@ class List < ActiveRecord::Base label? ? label.name : list_type.humanize end + def as_json(options = {}) + super(options).tap do |json| + if options.has_key?(:label) + json[:label] = label.as_json( + project: board.project, + only: [:id, :title, :description, :color] + ) + end + end + end + private def can_be_destroyed diff --git a/app/models/member.rb b/app/models/member.rb index 69406379948..df93aaee847 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -80,49 +80,77 @@ class Member < ActiveRecord::Base find_by(invite_token: invite_token) end - # This method is used to find users that have been entered into the "Add members" field. - # These can be the User objects directly, their IDs, their emails, or new emails to be invited. - def user_for_id(user_id) - return user_id if user_id.is_a?(User) - - user = User.find_by(id: user_id) - user ||= User.find_by(email: user_id) - user ||= user_id - user - end - - def add_user(members, user_id, access_level, current_user: nil, expires_at: nil) - user = user_for_id(user_id) + def add_user(source, user, access_level, current_user: nil, expires_at: nil) + user = retrieve_user(user) + access_level = retrieve_access_level(access_level) # `user` can be either a User object or an email to be invited - if user.is_a?(User) - member = members.find_or_initialize_by(user_id: user.id) + member = + if user.is_a?(User) + source.members.find_by(user_id: user.id) || + source.requesters.find_by(user_id: user.id) || + source.members.build(user_id: user.id) + else + source.members.build(invite_email: user) + end + + return member unless can_update_member?(current_user, member) + + member.attributes = { + created_by: member.created_by || current_user, + access_level: access_level, + expires_at: expires_at + } + + if member.request? + ::Members::ApproveAccessRequestService.new( + source, + current_user, + id: member.id, + access_level: access_level + ).execute else - member = members.build - member.invite_email = user + member.save end - if can_update_member?(current_user, member) || project_creator?(member, access_level) - member.created_by ||= current_user - member.access_level = access_level - member.expires_at = expires_at + UserProjectAccessChangedService.new(user.id).execute if user.is_a?(User) - member.save - end + member + end + + def access_levels + Gitlab::Access.sym_options end private + # This method is used to find users that have been entered into the "Add members" field. + # These can be the User objects directly, their IDs, their emails, or new emails to be invited. + def retrieve_user(user) + return user if user.is_a?(User) + + User.find_by(id: user) || User.find_by(email: user) || user + end + + def retrieve_access_level(access_level) + access_levels.fetch(access_level) { access_level.to_i } + end + def can_update_member?(current_user, member) # There is no current user for bulk actions, in which case anything is allowed - !current_user || - current_user.can?(:update_group_member, member) || - current_user.can?(:update_project_member, member) + !current_user || current_user.can?(:"update_#{member.type.underscore}", member) end - def project_creator?(member, access_level) - member.new_record? && member.owner? && - access_level.to_i == ProjectMember::MASTER + def add_users_to_source(source, users, access_level, current_user: nil, expires_at: nil) + users.each do |user| + add_user( + source, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) + end end end @@ -213,17 +241,28 @@ class Member < ActiveRecord::Base end def post_create_hook + UserProjectAccessChangedService.new(user.id).execute system_hook_service.execute_hooks_for(self, :create) end def post_update_hook - # override in subclass + UserProjectAccessChangedService.new(user.id).execute if access_level_changed? end def post_destroy_hook + refresh_member_authorized_projects system_hook_service.execute_hooks_for(self, :destroy) end + def refresh_member_authorized_projects + # If user/source is being destroyed, project access are gonna be destroyed eventually + # because of DB foreign keys, so we shouldn't bother with refreshing after each + # member is destroyed through association + return if destroyed_by_association.present? + + UserProjectAccessChangedService.new(user_id).execute + end + def after_accept_invite post_create_hook end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 2f13d339c89..204f34f0269 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -1,7 +1,7 @@ class GroupMember < Member SOURCE_TYPE = 'Namespace' - belongs_to :group, class_name: 'Group', foreign_key: 'source_id' + belongs_to :group, foreign_key: 'source_id' # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE @@ -12,6 +12,22 @@ class GroupMember < Member Gitlab::Access.options_with_owner end + def self.access_levels + Gitlab::Access.sym_options_with_owner + end + + def self.add_users_to_group(group, users, access_level, current_user: nil, expires_at: nil) + self.transaction do + add_users_to_source( + group, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) + end + end + def group source end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index ec2d40eb11c..008fff0857c 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -3,7 +3,7 @@ class ProjectMember < Member include Gitlab::ShellAdapter - belongs_to :project, class_name: 'Project', foreign_key: 'source_id' + belongs_to :project, foreign_key: 'source_id' # Make sure project member points only to project as it source default_value_for :source_type, SOURCE_TYPE @@ -34,36 +34,20 @@ class ProjectMember < Member # :master # ) # - def add_users_to_projects(project_ids, user_ids, access, current_user: nil, expires_at: nil) - access_level = if roles_hash.has_key?(access) - roles_hash[access] - elsif roles_hash.values.include?(access.to_i) - access - else - raise "Non valid access" - end - - users = user_ids.map { |user_id| Member.user_for_id(user_id) } - - ProjectMember.transaction do + def add_users_to_projects(project_ids, users, access_level, current_user: nil, expires_at: nil) + self.transaction do project_ids.each do |project_id| project = Project.find(project_id) - users.each do |user| - Member.add_user( - project.project_members, - user, - access_level, - current_user: current_user, - expires_at: expires_at - ) - end + add_users_to_source( + project, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) end end - - true - rescue - false end def truncate_teams(project_ids) @@ -84,13 +68,15 @@ class ProjectMember < Member truncate_teams [project.id] end - def roles_hash - Gitlab::Access.sym_options - end - def access_level_roles Gitlab::Access.options end + + private + + def can_update_member?(current_user, member) + super || (member.owner? && member.new_record?) + end end def access_field @@ -135,7 +121,11 @@ class ProjectMember < Member end def post_destroy_hook - event_service.leave_project(self.project, self.user) + if expired? + event_service.expired_leave_project(self.project, self.user) + else + event_service.leave_project(self.project, self.user) + end super end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f7d1253d957..fdf54cc8a7e 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -3,11 +3,10 @@ class MergeRequest < ActiveRecord::Base include Issuable include Referable include Sortable - include Taskable include Importable - belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project" - belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project" + belongs_to :target_project, class_name: "Project" + belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" has_many :merge_request_diffs, dependent: :destroy @@ -16,6 +15,8 @@ class MergeRequest < ActiveRecord::Base has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + serialize :merge_params, Hash after_create :ensure_merge_request_diff, unless: :importing? @@ -29,7 +30,7 @@ class MergeRequest < ActiveRecord::Base # Temporary fields to store compare vars # when creating new merge request - attr_accessor :can_be_created, :compare_commits, :compare + attr_accessor :can_be_created, :compare_commits, :diff_options, :compare state_machine :state, initial: :opened do event :close do @@ -135,6 +136,10 @@ class MergeRequest < ActiveRecord::Base reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end + def self.project_foreign_key + 'target_project_id' + end + # Returns all the merge requests from an ActiveRecord:Relation. # # This method uses a UNION as it usually operates on the result of @@ -153,6 +158,20 @@ class MergeRequest < ActiveRecord::Base where("merge_requests.id IN (#{union.to_sql})") end + WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze + + def self.work_in_progress?(title) + !!(title =~ WIP_REGEX) + end + + def self.wipless_title(title) + title.sub(WIP_REGEX, "") + end + + def self.wip_title(title) + work_in_progress?(title) ? title : "WIP: #{title}" + end + def to_reference(from_project = nil) reference = "#{self.class.reference_prefix}#{iid}" @@ -180,7 +199,7 @@ class MergeRequest < ActiveRecord::Base end def diff_size - merge_request_diff.size + diffs(diff_options).size end def diff_base_commit @@ -306,21 +325,17 @@ class MergeRequest < ActiveRecord::Base def validate_fork return true unless target_project && source_project return true if target_project == source_project - return true unless forked_source_project_missing? + return true unless source_project_missing? errors.add :validate_fork, 'Source project is not a fork of the target project' end def closed_without_fork? - closed? && forked_source_project_missing? - end - - def closed_without_source_project? - closed? && !source_project + closed? && source_project_missing? end - def forked_source_project_missing? + def source_project_missing? return false unless for_fork? return true unless source_project @@ -328,9 +343,7 @@ class MergeRequest < ActiveRecord::Base end def reopenable? - return false if closed_without_fork? || closed_without_source_project? || merged? - - closed? + closed? && !source_project_missing? && source_branch_exists? end def ensure_merge_request_diff @@ -387,14 +400,16 @@ class MergeRequest < ActiveRecord::Base @closed_event ||= target_project.events.where(target_id: self.id, target_type: "MergeRequest", action: Event::CLOSED).last end - WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze - def work_in_progress? - !!(title =~ WIP_REGEX) + self.class.work_in_progress?(title) end def wipless_title - self.title.sub(WIP_REGEX, "") + self.class.wipless_title(self.title) + end + + def wip_title + self.class.wip_title(self.title) end def mergeable?(skip_ci_check: false) @@ -410,6 +425,7 @@ class MergeRequest < ActiveRecord::Base return false if work_in_progress? return false if broken? return false unless skip_ci_check || mergeable_ci_state? + return false unless mergeable_discussions_state? true end @@ -426,11 +442,11 @@ class MergeRequest < ActiveRecord::Base end def should_remove_source_branch? - merge_params['should_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['should_remove_source_branch']) end def force_remove_source_branch? - merge_params['force_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['force_remove_source_branch']) end def remove_source_branch? @@ -478,6 +494,16 @@ class MergeRequest < ActiveRecord::Base discussions_resolvable? && diff_discussions.none?(&:to_be_resolved?) end + def discussions_to_be_resolved? + discussions_resolvable? && !discussions_resolved? + end + + def mergeable_discussions_state? + return true unless project.only_allow_merge_if_all_discussions_are_resolved? + + !discussions_to_be_resolved? + end + def hook_attrs attrs = { source: source_project.try(:hook_attrs), @@ -501,6 +527,23 @@ class MergeRequest < ActiveRecord::Base target_project end + # If the merge request closes any issues, save this information in the + # `MergeRequestsClosingIssues` model. This is a performance optimization. + # Calculating this information for a number of merge requests requires + # running `ReferenceExtractor` on each of them separately. + # This optimization does not apply to issues from external sources. + def cache_merge_request_closes_issues!(current_user = self.author) + return if project.has_external_issue_tracker? + + transaction do + self.merge_requests_closing_issues.delete_all + + closes_issues(current_user).each do |issue| + self.merge_requests_closing_issues.create!(issue: issue) + end + end + end + def closes_issue?(issue) closes_issues.include?(issue) end @@ -508,7 +551,8 @@ class MergeRequest < ActiveRecord::Base # Return the set of issues that will be closed if this merge request is accepted. def closes_issues(current_user = self.author) if target_branch == project.default_branch - messages = commits.map(&:safe_message) << description + messages = [description] + messages.concat(commits.map(&:safe_message)) if merge_request_diff Gitlab::ClosingIssueExtractor.new(project, current_user). closed_by_message(messages.join("\n")) @@ -574,13 +618,11 @@ class MergeRequest < ActiveRecord::Base end def merge_commit_message - message = "Merge branch '#{source_branch}' into '#{target_branch}'" - message << "\n\n" - message << title.to_s - message << "\n\n" - message << description.to_s - message << "\n\n" - message << "See merge request !#{iid}" + message = "Merge branch '#{source_branch}' into '#{target_branch}'\n\n" + message << "#{title}\n\n" + message << "#{description}\n\n" if description.present? + message << "See merge request #{to_reference}" + message end @@ -624,7 +666,7 @@ class MergeRequest < ActiveRecord::Base end def has_ci? - source_project.ci_service && commits.any? + source_project.try(:ci_service) && commits.any? end def branch_missing? @@ -648,14 +690,20 @@ class MergeRequest < ActiveRecord::Base def mergeable_ci_state? return true unless project.only_allow_merge_if_build_succeeds? - !pipeline || pipeline.success? + !pipeline || pipeline.success? || pipeline.skipped? end def environments - return unless diff_head_commit + return [] unless diff_head_commit - target_project.environments.select do |environment| - environment.includes_commit?(diff_head_commit) + @environments ||= begin + target_envs = target_project.environments_for( + target_branch, commit: diff_head_commit, with_tags: true) + + source_envs = source_project.environments_for( + source_branch, commit: diff_head_commit) if source_project + + (target_envs.to_a + source_envs.to_a).uniq end end @@ -745,10 +793,23 @@ class MergeRequest < ActiveRecord::Base end def all_pipelines - @all_pipelines ||= - if diff_head_sha && source_project - source_project.pipelines.order(id: :desc).where(sha: commits_sha, ref: source_branch) - end + return unless source_project + + @all_pipelines ||= source_project.pipelines + .where(sha: all_commits_sha, ref: source_branch) + .order(id: :desc) + end + + # Note that this could also return SHA from now dangling commits + # + def all_commits_sha + if persisted? + merge_request_diffs.flat_map(&:commits_sha).uniq + elsif compare_commits + compare_commits.to_a.reverse.map(&:id) + else + [diff_head_sha] + end end def merge_commit @@ -818,7 +879,7 @@ class MergeRequest < ActiveRecord::Base # files. conflicts.files.each(&:lines) @conflicts_can_be_resolved_in_ui = conflicts.files.length > 0 - rescue Rugged::OdbError, Gitlab::Conflict::Parser::ParserError, Gitlab::Conflict::FileCollection::ConflictSideMissing + rescue Rugged::OdbError, Gitlab::Conflict::Parser::UnresolvableError, Gitlab::Conflict::FileCollection::ConflictSideMissing @conflicts_can_be_resolved_in_ui = false end end diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb new file mode 100644 index 00000000000..cdc408738be --- /dev/null +++ b/app/models/merge_request/metrics.rb @@ -0,0 +1,12 @@ +class MergeRequest::Metrics < ActiveRecord::Base + belongs_to :merge_request + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :pipeline_id + + def record! + if merge_request.merged? && self.merged_at.blank? + self.merged_at = Time.now + end + + self.save + end +end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 18c583add88..58a24eb84cb 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -6,8 +6,14 @@ class MergeRequestDiff < ActiveRecord::Base # Prevent store of diff if commits amount more then 500 COMMITS_SAFE_SIZE = 100 + # Valid types of serialized diffs allowed by Gitlab::Git::Diff + VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta] + belongs_to :merge_request + serialize :st_commits + serialize :st_diffs + state_machine :state, initial: :empty do state :collected state :overflow @@ -19,8 +25,7 @@ class MergeRequestDiff < ActiveRecord::Base state :overflow_diff_lines_limit end - serialize :st_commits - serialize :st_diffs + scope :viewable, -> { without_state(:empty) } # All diff information is collected from repository after object is created. # It allows you to override variables like head_commit_sha before getting diff. @@ -30,6 +35,10 @@ class MergeRequestDiff < ActiveRecord::Base select(column_names - ['st_diffs']) end + def st_commits + super || [] + end + # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -83,7 +92,7 @@ class MergeRequestDiff < ActiveRecord::Base end def commits - @commits ||= load_commits(st_commits || []) + @commits ||= load_commits(st_commits) end def reload_commits @@ -117,6 +126,14 @@ class MergeRequestDiff < ActiveRecord::Base project.commit(head_commit_sha) end + def commits_sha + if @commits + commits.map(&:sha) + else + st_commits.map { |commit| commit[:id] } + end + end + def diff_refs return unless start_commit_sha || base_commit_sha @@ -152,12 +169,24 @@ class MergeRequestDiff < ActiveRecord::Base self == merge_request.merge_request_diff end - def compare_with(sha) - CompareService.new.execute(project, head_commit_sha, project, sha) + def compare_with(sha, straight: true) + # When compare merge request versions we want diff A..B instead of A...B + # so we handle cases when user does squash and rebase of the commits between versions. + # For this reason we set straight to true by default. + CompareService.new.execute(project, head_commit_sha, project, sha, straight: straight) end private + # Old GitLab implementations may have generated diffs as ["--broken-diff"]. + # Avoid an error 500 by ignoring bad elements. See: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776 + def valid_raw_diff?(raw) + return false unless raw.respond_to?(:each) + + raw.any? { |element| VALID_CLASSES.include?(element.class) } + end + def dump_commits(commits) commits.map(&:to_hash) end @@ -188,7 +217,7 @@ class MergeRequestDiff < ActiveRecord::Base end def load_diffs(raw, options) - if raw.respond_to?(:each) + if valid_raw_diff?(raw) if paths = options[:paths] raw = raw.select do |diff| paths.include?(diff[:old_path]) || paths.include?(diff[:new_path]) @@ -272,8 +301,10 @@ class MergeRequestDiff < ActiveRecord::Base end def keep_around_commits - repository.keep_around(start_commit_sha) - repository.keep_around(head_commit_sha) - repository.keep_around(base_commit_sha) + [repository, merge_request.source_project.repository].each do |repo| + repo.keep_around(start_commit_sha) + repo.keep_around(head_commit_sha) + repo.keep_around(base_commit_sha) + end end end diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb new file mode 100644 index 00000000000..ab597c37947 --- /dev/null +++ b/app/models/merge_requests_closing_issues.rb @@ -0,0 +1,7 @@ +class MergeRequestsClosingIssues < ActiveRecord::Base + belongs_to :merge_request + belongs_to :issue + + validates :merge_request_id, uniqueness: { scope: :issue_id }, presence: true + validates :issue_id, presence: true +end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 2bd7f198030..c774e69080c 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -6,12 +6,16 @@ class Milestone < ActiveRecord::Base Any = MilestoneStruct.new('Any Milestone', '', -1) Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2) + include CacheMarkdownField include InternalId include Sortable include Referable include StripAttribute include Milestoneish + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description + belongs_to :project has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues @@ -25,6 +29,7 @@ class Milestone < ActiveRecord::Base validates :title, presence: true, uniqueness: { scope: :project_id } validates :project, presence: true + validate :start_date_should_be_less_than_due_date, if: Proc.new { |m| m.start_date.present? && m.due_date.present? } strip_attributes :title @@ -127,24 +132,6 @@ class Milestone < ActiveRecord::Base self.title end - def expired? - if due_date - due_date.past? - else - false - end - end - - def expires_at - if due_date - if due_date.past? - "expired on #{due_date.to_s(:medium)}" - else - "expires on #{due_date.to_s(:medium)}" - end - end - end - def can_be_closed? active? && issues.opened.count.zero? end @@ -158,7 +145,7 @@ class Milestone < ActiveRecord::Base end def title=(value) - write_attribute(:title, Sanitize.clean(value.to_s)) if value.present? + write_attribute(:title, sanitize_title(value)) if value.present? end # Sorts the issues for the given IDs. @@ -204,4 +191,14 @@ class Milestone < ActiveRecord::Base iid end end + + def sanitize_title(value) + CGI.unescape_html(Sanitize.clean(value.to_s)) + end + + def start_date_should_be_less_than_due_date + if due_date <= start_date + errors.add(:start_date, "Can't be greater than due date") + end + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 919b3b1f095..891dffac648 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -1,9 +1,12 @@ class Namespace < ActiveRecord::Base acts_as_paranoid + include CacheMarkdownField include Sortable include Gitlab::ShellAdapter + cache_markdown_field :description, pipeline: :description + has_many :projects, dependent: :destroy belongs_to :owner, class_name: "User" @@ -24,6 +27,7 @@ class Namespace < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true after_update :move_dir, if: :path_changed? + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } # Save the storage paths before the projects are destroyed to use them on after destroy before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths } @@ -58,15 +62,13 @@ class Namespace < ActiveRecord::Base def clean_path(path) path = path.dup # Get the email username by removing everything after an `@` sign. - path.gsub!(/@.*\z/, "") - # Usernames can't end in .git, so remove it. - path.gsub!(/\.git\z/, "") - # Remove dashes at the start of the username. - path.gsub!(/\A-+/, "") - # Remove periods at the end of the username. - path.gsub!(/\.+\z/, "") + path.gsub!(/@.*\z/, "") # Remove everything that's not in the list of allowed characters. - path.gsub!(/[^a-zA-Z0-9_\-\.]/, "") + path.gsub!(/[^a-zA-Z0-9_\-\.]/, "") + # Remove trailing violations ('.atom', '.git', or '.') + path.gsub!(/(\.atom|\.git|\.)*\z/, "") + # Remove leading violations ('-') + path.gsub!(/\A\-+/, "") # Users with the great usernames of "." or ".." would end up with a blank username. # Work around that by setting their username to "blank", followed by a counter. @@ -102,6 +104,8 @@ class Namespace < ActiveRecord::Base gitlab_shell.add_namespace(repository_storage_path, path_was) unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) + Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs raise Exception.new('namespace directory cannot be moved') @@ -174,4 +178,11 @@ class Namespace < ActiveRecord::Base end end end + + def refresh_access_of_projects_invited_groups + Group. + joins(project_group_links: :project). + where(projects: { namespace_id: id }). + find_each(&:refresh_members_authorized_projects) + end end diff --git a/app/models/note.rb b/app/models/note.rb index b94e3cff2ce..ed4224e3046 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -6,15 +6,22 @@ class Note < ActiveRecord::Base include Awardable include Importable include FasterCacheKeys + include CacheMarkdownField + include AfterCommitQueue + + cache_markdown_field :note, pipeline: :note # Attribute containing rendered and redacted Markdown as generated by # Banzai::ObjectRenderer. - attr_accessor :note_html + attr_accessor :redacted_note_html # An Array containing the number of visible references as generated by # Banzai::ObjectRenderer attr_accessor :user_visible_reference_count + # Attribute used to store the attributes that have ben changed by slash commands. + attr_accessor :commands_changes + default_value_for :system, false attr_mentionable :note, pipeline: :note @@ -223,10 +230,6 @@ class Note < ActiveRecord::Base end end - def user_authored?(user) - user == author - end - def award_emoji? can_be_award_emoji? && contains_emoji_only? end diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 121b598b8f3..43fc218de2b 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -32,7 +32,9 @@ class NotificationSetting < ActiveRecord::Base :reopen_merge_request, :close_merge_request, :reassign_merge_request, - :merge_merge_request + :merge_merge_request, + :failed_pipeline, + :success_pipeline ] store :events, accessors: EMAIL_EVENTS, coder: JSON diff --git a/app/models/project.rb b/app/models/project.rb index 8b5a6f167bd..9256e9ddd95 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -6,28 +6,43 @@ class Project < ActiveRecord::Base include Gitlab::VisibilityLevel include Gitlab::CurrentSettings include AccessRequestable + include CacheMarkdownField include Referable include Sortable include AfterCommitQueue include CaseSensitivity include TokenAuthenticatable include ProjectFeaturesCompatibility + include SelectForProjectAuthorization extend Gitlab::ConfigHelper + class BoardLimitExceeded < StandardError; end + + NUMBER_OF_PERMITTED_BOARDS = 1 UNKNOWN_IMPORT_URL = 'http://unknown.git' - delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true + cache_markdown_field :description, pipeline: :description + + delegate :feature_available?, :builds_enabled?, :wiki_enabled?, + :merge_requests_enabled?, :issues_enabled?, to: :project_feature, + allow_nil: true default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :container_registry_enabled, gitlab_config_features.container_registry - default_value_for(:repository_storage) { current_application_settings.repository_storage } + default_value_for(:repository_storage) { current_application_settings.pick_repository_storage } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } + default_value_for :issues_enabled, gitlab_config_features.issues + default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests + default_value_for :builds_enabled, gitlab_config_features.builds + default_value_for :wiki_enabled, gitlab_config_features.wiki + default_value_for :snippets_enabled, gitlab_config_features.snippets + default_value_for :only_allow_merge_if_all_discussions_are_resolved, false after_create :ensure_dir_exist + after_create :create_project_feature, unless: :project_feature after_save :ensure_dir_exist, if: :namespace_id_changed? - after_initialize :setup_project_feature # set last_activity_at to the same as created_at after_create :set_last_activity_at @@ -57,20 +72,20 @@ class Project < ActiveRecord::Base alias_attribute :title, :name # Relations - belongs_to :creator, foreign_key: 'creator_id', class_name: 'User' + belongs_to :creator, class_name: 'User' belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' belongs_to :namespace - has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event', foreign_key: 'project_id' - - has_one :board, dependent: :destroy + has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' + has_many :boards, before_add: :validate_board_limit, dependent: :destroy + has_many :chat_services # Project services - has_many :services has_one :campfire_service, dependent: :destroy has_one :drone_ci_service, dependent: :destroy has_one :emails_on_push_service, dependent: :destroy has_one :builds_email_service, dependent: :destroy + has_one :pipelines_email_service, dependent: :destroy has_one :irker_service, dependent: :destroy has_one :pivotaltracker_service, dependent: :destroy has_one :hipchat_service, dependent: :destroy @@ -78,6 +93,7 @@ class Project < ActiveRecord::Base has_one :assembla_service, dependent: :destroy has_one :asana_service, dependent: :destroy has_one :gemnasium_service, dependent: :destroy + has_one :mattermost_slash_commands_service, dependent: :destroy has_one :slack_service, dependent: :destroy has_one :buildkite_service, dependent: :destroy has_one :bamboo_service, dependent: :destroy @@ -101,7 +117,7 @@ class Project < ActiveRecord::Base # Merge requests from source project should be kept when source project was removed has_many :fork_merge_requests, foreign_key: 'source_project_id', class_name: MergeRequest has_many :issues, dependent: :destroy - has_many :labels, dependent: :destroy + has_many :labels, dependent: :destroy, class_name: 'ProjectLabel' has_many :services, dependent: :destroy has_many :events, dependent: :destroy has_many :milestones, dependent: :destroy @@ -110,7 +126,9 @@ class Project < ActiveRecord::Base has_many :hooks, dependent: :destroy, class_name: 'ProjectHook' has_many :protected_branches, dependent: :destroy - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'ProjectMember' + has_many :project_authorizations, dependent: :destroy + has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' + has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :project_members has_many :users, through: :project_members @@ -131,7 +149,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy - has_many :commit_statuses, dependent: :destroy, class_name: 'CommitStatus', foreign_key: :gl_project_id + has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id has_many :builds, class_name: 'Ci::Build', foreign_key: :gl_project_id # the builds are created from the commit_statuses has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject', foreign_key: :gl_project_id @@ -146,6 +164,8 @@ class Project < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true + delegate :add_user, to: :team + delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team # Validations validates :creator, presence: true, on: :create @@ -157,6 +177,7 @@ class Project < ActiveRecord::Base message: Gitlab::Regex.project_name_regex_message } validates :path, presence: true, + project_path: true, length: { within: 0..255 }, format: { with: Gitlab::Regex.project_path_regex, message: Gitlab::Regex.project_path_regex_message } @@ -195,8 +216,38 @@ class Project < ActiveRecord::Base scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } - scope :with_builds_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') } - scope :with_issues_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.issues_access_level IS NULL or project_features.issues_access_level > 0') } + scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } + + # "enabled" here means "not disabled". It includes private features! + scope :with_feature_enabled, ->(feature) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => [nil, ProjectFeature::PRIVATE, ProjectFeature::ENABLED] }) + } + + # Picks a feature where the level is exactly that given. + scope :with_feature_access_level, ->(feature, level) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => level }) + } + + scope :with_builds_enabled, -> { with_feature_enabled(:builds) } + scope :with_issues_enabled, -> { with_feature_enabled(:issues) } + + # project features may be "disabled", "internal" or "enabled". If "internal", + # they are only available to team members. This scope returns projects where + # the feature is either enabled, or internal with permission for the user. + def self.with_feature_available_for_user(feature, user) + return with_feature_enabled(feature) if user.try(:admin?) + + unconditional = with_feature_access_level(feature, [nil, ProjectFeature::ENABLED]) + return unconditional if user.nil? + + conditional = with_feature_access_level(feature, ProjectFeature::PRIVATE) + authorized = user.authorized_projects.merge(conditional.reorder(nil)) + + union = Gitlab::SQL::Union.new([unconditional.select(:id), authorized.select(:id)]) + where(arel_table[:id].in(Arel::Nodes::SqlLiteral.new(union.to_sql))) + end scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } @@ -371,18 +422,9 @@ class Project < ActiveRecord::Base %r{(?<project>#{name_pattern}/#{name_pattern})} end - def trending(since = 1.month.ago) - # By counting in the JOIN we don't expose the GROUP BY to the outer query. - # This means that calls such as "any?" and "count" just return a number of - # the total count, instead of the counts grouped per project as a Hash. - join_body = "INNER JOIN ( - SELECT project_id, COUNT(*) AS amount - FROM notes - WHERE created_at >= #{sanitize(since)} - GROUP BY project_id - ) join_note_counts ON projects.id = join_note_counts.project_id" - - joins(join_body).reorder('join_note_counts.amount DESC') + def trending + joins('INNER JOIN trending_projects ON projects.id = trending_projects.project_id'). + reorder('trending_projects.id ASC') end def cached_count @@ -390,6 +432,10 @@ class Project < ActiveRecord::Base Project.count end end + + def group_ids + joins(:namespace).where(namespaces: { type: 'Group' }).select(:namespace_id) + end end def lfs_enabled? @@ -493,7 +539,7 @@ class Project < ActiveRecord::Base end def import_url - if import_data && super + if import_data && super.present? import_url = Gitlab::UrlSanitizer.new(super, credentials: import_data.credentials) import_url.full_url else @@ -617,13 +663,12 @@ class Project < ActiveRecord::Base end def new_issue_address(author) - # This feature is disabled for the time being. - return nil + return unless Gitlab::IncomingEmail.supports_issue_creation? && author - if Gitlab::IncomingEmail.enabled? && author # rubocop:disable Lint/UnreachableCode - Gitlab::IncomingEmail.reply_address( - "#{path_with_namespace}+#{author.authentication_token}") - end + author.ensure_incoming_email_token! + + Gitlab::IncomingEmail.reply_address( + "#{path_with_namespace}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -666,6 +711,10 @@ class Project < ActiveRecord::Base end end + def issue_reference_pattern + issues_tracker.reference_pattern + end + def default_issues_tracker? !external_issue_tracker end @@ -708,33 +757,36 @@ class Project < ActiveRecord::Base update_column(:has_external_wiki, services.external_wikis.any?) end - def build_missing_services + def find_or_initialize_services services_templates = Service.where(template: true) - Service.available_services_names.each do |service_name| + Service.available_services_names.map do |service_name| service = find_service(services, service_name) - # If service is available but missing in db - if service.nil? + if service + service + else # We should check if template for the service exists template = find_service(services_templates, service_name) if template.nil? - # If no template, we should create an instance. Ex `create_gitlab_ci_service` - self.send :"create_#{service_name}_service" + # If no template, we should create an instance. Ex `build_gitlab_ci_service` + public_send("build_#{service_name}_service") else - Service.create_from_template(self.id, template) + Service.build_from_template(id, template) end end end end + def find_or_initialize_service(name) + find_or_initialize_services.find { |service| service.to_param == name } + end + def create_labels Label.templates.each do |label| - label = label.dup - label.template = nil - label.project_id = self.id - label.save + params = label.attributes.except('id', 'template', 'created_at', 'updated_at') + Labels::FindOrCreateService.new(nil, self, params).execute(skip_authorization: true) end end @@ -832,11 +884,6 @@ class Project < ActiveRecord::Base end end - def update_merge_requests(oldrev, newrev, ref, user) - MergeRequests::RefreshService.new(self, user). - execute(oldrev, newrev, ref) - end - def valid_repo? repository.exists? rescue @@ -845,7 +892,7 @@ class Project < ActiveRecord::Base end def empty_repo? - !repository.exists? || !repository.has_visible_content? + repository.empty_repo? end def repo @@ -1016,10 +1063,6 @@ class Project < ActiveRecord::Base project_members.find_by(user_id: user) end - def add_user(user, access_level, current_user: nil, expires_at: nil) - team.add_user(user, access_level, current_user: current_user, expires_at: expires_at) - end - def default_branch @default_branch ||= repository.root_ref if repository.exists? end @@ -1047,7 +1090,7 @@ class Project < ActiveRecord::Base "refs/heads/#{branch}", force: true) repository.copy_gitattributes(branch) - repository.expire_avatar_cache(branch) + repository.expire_avatar_cache reload_default_branch end @@ -1067,10 +1110,6 @@ class Project < ActiveRecord::Base forks.count end - def find_label(name) - labels.find_by(name: name) - end - def origin_merge_requests merge_requests.where(source_project_id: self.id) end @@ -1137,12 +1176,6 @@ class Project < ActiveRecord::Base self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - # TODO (ayufan): For now we use runners_token (backward compatibility) - # In 8.4 every build will have its own individual token valid for time of build - def valid_build_token?(token) - self.builds_enabled? && self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) - end - def build_coverage_enabled? build_coverage_regex.present? end @@ -1263,20 +1296,6 @@ class Project < ActiveRecord::Base end end - # Checks if `user` is authorized for this project, with at least the - # `min_access_level` (if given). - # - # If you change the logic of this method, please also update `User#authorized_projects` - def authorized_for_user?(user, min_access_level = nil) - return false unless user - - return true if personal? && namespace_id == user.namespace_id - - authorized_for_user_by_group?(user, min_access_level) || - authorized_for_user_by_members?(user, min_access_level) || - authorized_for_user_by_shared_projects?(user, min_access_level) - end - def append_or_update_attribute(name, value) old_values = public_send(name.to_s) @@ -1299,43 +1318,49 @@ class Project < ActiveRecord::Base Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } end - private + def environments_for(ref, commit: nil, with_tags: false) + deployments_query = with_tags ? 'ref = ? OR tag IS TRUE' : 'ref = ?' - def pushes_since_gc_redis_key - "projects/#{id}/pushes_since_gc" - end + environment_ids = deployments + .where(deployments_query, ref.to_s) + .group(:environment_id) + .select(:environment_id) - # Prevents the creation of project_feature record for every project - def setup_project_feature - build_project_feature unless project_feature - end + environments_found = environments.available + .where(id: environment_ids).to_a - def default_branch_protected? - current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || - current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE - end + return environments_found unless commit - def authorized_for_user_by_group?(user, min_access_level) - member = user.group_members.find_by(source_id: group) + environments_found.select do |environment| + environment.includes_commit?(commit) + end + end - member && (!min_access_level || member.access_level >= min_access_level) + def environments_recently_updated_on_branch(branch) + environments_for(branch).select do |environment| + environment.recently_updated_on_branch?(branch) + end end - def authorized_for_user_by_members?(user, min_access_level) - member = members.find_by(user_id: user) + private - member && (!min_access_level || member.access_level >= min_access_level) + def pushes_since_gc_redis_key + "projects/#{id}/pushes_since_gc" end - def authorized_for_user_by_shared_projects?(user, min_access_level) - shared_projects = user.group_members.joins(group: :shared_projects). - where(project_group_links: { project_id: self }) - - if min_access_level - members_scope = { access_level: Gitlab::Access.values.select { |access| access >= min_access_level } } - shared_projects = shared_projects.where(members: members_scope) - end + def default_branch_protected? + current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || + current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE + end - shared_projects.any? + # Similar to the normal callbacks that hook into the life cycle of an + # Active Record object, you can also define callbacks that get triggered + # when you add an object to an association collection. If any of these + # callbacks throw an exception, the object will not be added to the + # collection. Before you add a new board to the boards collection if you + # already have 1, 2, or n it will fail, but it if you have 0 that is lower + # than the number of permitted boards per project it won't fail. + def validate_board_limit(board) + raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS end end diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb new file mode 100644 index 00000000000..a00d43773d9 --- /dev/null +++ b/app/models/project_authorization.rb @@ -0,0 +1,8 @@ +class ProjectAuthorization < ActiveRecord::Base + belongs_to :user + belongs_to :project + + validates :project, presence: true + validates :access_level, inclusion: { in: Gitlab::Access.all_values }, presence: true + validates :user, uniqueness: { scope: [:project, :access_level] }, presence: true +end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 9c602c582bd..03194fc2141 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -13,41 +13,71 @@ class ProjectFeature < ActiveRecord::Base # Enabled: enabled for everyone able to access the project # - # Permision levels + # Permission levels DISABLED = 0 PRIVATE = 10 ENABLED = 20 - FEATURES = %i(issues merge_requests wiki snippets builds) + FEATURES = %i(issues merge_requests wiki snippets builds repository) - belongs_to :project + class << self + def access_level_attribute(feature) + feature = feature.model_name.plural.to_sym if feature.respond_to?(:model_name) + raise ArgumentError, "invalid project feature: #{feature}" unless FEATURES.include?(feature) - def feature_available?(feature, user) - raise ArgumentError, 'invalid project feature' unless FEATURES.include?(feature) + "#{feature}_access_level".to_sym + end + end + + # Default scopes force us to unscope here since a service may need to check + # permissions for a project in pending_delete + # http://stackoverflow.com/questions/1540645/how-to-disable-default-scope-for-a-belongs-to + belongs_to :project, -> { unscope(where: :pending_delete) } - get_permission(user, public_send("#{feature}_access_level")) + validate :repository_children_level + + default_value_for :builds_access_level, value: ENABLED, allows_nil: false + default_value_for :issues_access_level, value: ENABLED, allows_nil: false + default_value_for :merge_requests_access_level, value: ENABLED, allows_nil: false + default_value_for :snippets_access_level, value: ENABLED, allows_nil: false + default_value_for :wiki_access_level, value: ENABLED, allows_nil: false + default_value_for :repository_access_level, value: ENABLED, allows_nil: false + + def feature_available?(feature, user) + access_level = public_send(ProjectFeature.access_level_attribute(feature)) + get_permission(user, access_level) end def builds_enabled? - return true unless builds_access_level - builds_access_level > DISABLED end def wiki_enabled? - return true unless wiki_access_level - wiki_access_level > DISABLED end def merge_requests_enabled? - return true unless merge_requests_access_level - merge_requests_access_level > DISABLED end + def issues_enabled? + issues_access_level > DISABLED + end + private + # Validates builds and merge requests access level + # which cannot be higher than repository access level + def repository_children_level + validator = lambda do |field| + level = public_send(field) || ProjectFeature::ENABLED + not_allowed = level > repository_access_level + self.errors.add(field, "cannot have higher visibility level than repository access level") if not_allowed + end + + %i(merge_requests_access_level builds_access_level).each(&validator) + end + def get_permission(user, level) case level when DISABLED diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index 7613cbdea93..6149c35cc61 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -10,12 +10,15 @@ class ProjectGroupLink < ActiveRecord::Base belongs_to :group validates :project_id, presence: true - validates :group_id, presence: true + validates :group, presence: true validates :group_id, uniqueness: { scope: [:project_id], message: "already shared with this group" } validates :group_access, presence: true validates :group_access, inclusion: { in: Gitlab::Access.values }, presence: true validate :different_group + after_create :refresh_group_members_authorized_projects + after_destroy :refresh_group_members_authorized_projects + def self.access_options Gitlab::Access.options end @@ -35,4 +38,8 @@ class ProjectGroupLink < ActiveRecord::Base errors.add(:base, "Project cannot be shared with the project it is in.") end end + + def refresh_group_members_authorized_projects + group.refresh_members_authorized_projects + end end diff --git a/app/models/project_label.rb b/app/models/project_label.rb new file mode 100644 index 00000000000..82f47f0e8fd --- /dev/null +++ b/app/models/project_label.rb @@ -0,0 +1,38 @@ +class ProjectLabel < Label + MAX_NUMBER_OF_PRIORITIES = 1 + + belongs_to :project + + validates :project, presence: true + + validate :permitted_numbers_of_priorities + validate :title_must_not_exist_at_group_level + + delegate :group, to: :project, allow_nil: true + + alias_attribute :subject, :project + + def subject_foreign_key + 'project_id' + end + + def to_reference(target_project = nil, format: :id) + super(project, target_project, format: format) + end + + private + + def title_must_not_exist_at_group_level + return unless group.present? && title_changed? + + if group.labels.with_title(self.title).exists? + errors.add(:title, :label_already_exists_at_group_level, group: group.name) + end + end + + def permitted_numbers_of_priorities + if priorities && priorities.size > MAX_NUMBER_OF_PRIORITIES + errors.add(:priorities, 'Number of permitted priorities exceeded') + end + end +end diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 81af55aa29a..338e685339a 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -1,4 +1,6 @@ class BugzillaService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title diff --git a/app/models/project_services/builds_email_service.rb b/app/models/project_services/builds_email_service.rb index fa66e5864b8..201b94b065b 100644 --- a/app/models/project_services/builds_email_service.rb +++ b/app/models/project_services/builds_email_service.rb @@ -43,7 +43,7 @@ class BuildsEmailService < Service end def can_test? - project.builds.count > 0 + project.builds.any? end def disabled_title diff --git a/app/models/project_services/chat_service.rb b/app/models/project_services/chat_service.rb new file mode 100644 index 00000000000..d36beff5fa6 --- /dev/null +++ b/app/models/project_services/chat_service.rb @@ -0,0 +1,21 @@ +# Base class for Chat services +# This class is not meant to be used directly, but only to inherrit from. +class ChatService < Service + default_value_for :category, 'chat' + + has_many :chat_names, foreign_key: :service_id + + def valid_token?(token) + self.respond_to?(:token) && + self.token.present? && + ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) + end + + def supported_events + [] + end + + def trigger(params) + raise NotImplementedError + end +end diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index 63a5ed14484..b2f426dc2ac 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -1,4 +1,6 @@ class CustomIssueTrackerService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title @@ -9,6 +11,10 @@ class CustomIssueTrackerService < IssueTrackerService end end + def title=(value) + self.properties['title'] = value if self.properties + end + def description if self.properties && self.properties['description'].present? self.properties['description'] diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index 5d17c358330..6bd8d4ec568 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -1,6 +1,8 @@ class GitlabIssueTrackerService < IssueTrackerService include Gitlab::Routing.url_helpers + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url default_value_for :default, true diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index afebd3b6a12..660a8ae3421 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -1,5 +1,12 @@ class HipchatService < Service + include ActionView::Helpers::SanitizeHelper + MAX_COMMITS = 3 + HIPCHAT_ALLOWED_TAGS = %w[ + a b i strong em br img pre code + table th tr td caption colgroup col thead tbody tfoot + ul ol li dl dt dd + ] prop_accessor :token, :room, :server, :notify, :color, :api_version boolean_accessor :notify_only_broken_builds @@ -88,6 +95,10 @@ class HipchatService < Service end end + def render_line(text) + markdown(text.lines.first.chomp, pipeline: :single_line) if text + end + def create_push_message(push) ref_type = Gitlab::Git.tag_ref?(push[:ref]) ? 'tag' : 'branch' ref = Gitlab::Git.ref_name(push[:ref]) @@ -110,7 +121,7 @@ class HipchatService < Service message << "(<a href=\"#{project.web_url}/compare/#{before}...#{after}\">Compare changes</a>)" push[:commits].take(MAX_COMMITS).each do |commit| - message << "<br /> - #{commit[:message].lines.first} (<a href=\"#{commit[:url]}\">#{commit[:id][0..5]}</a>)" + message << "<br /> - #{render_line(commit[:message])} (<a href=\"#{commit[:url]}\">#{commit[:id][0..5]}</a>)" end if push[:commits].count > MAX_COMMITS @@ -121,12 +132,22 @@ class HipchatService < Service message end - def format_body(body) - if body - body = body.truncate(200, separator: ' ', omission: '...') - end + def markdown(text, options = {}) + return "" unless text + + context = { + project: project, + pipeline: :email + } + + Banzai.render(text, context) - "<pre>#{body}</pre>" + context.merge!(options) + + html = Banzai.post_process(Banzai.render(text, context), context) + sanitized_html = sanitize(html, tags: HIPCHAT_ALLOWED_TAGS, attributes: %w[href title alt]) + + sanitized_html.truncate(200, separator: ' ', omission: '...') end def create_issue_message(data) @@ -134,7 +155,7 @@ class HipchatService < Service obj_attr = data[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) - title = obj_attr[:title] + title = render_line(obj_attr[:title]) state = obj_attr[:state] issue_iid = obj_attr[:iid] issue_url = obj_attr[:url] @@ -143,10 +164,7 @@ class HipchatService < Service issue_link = "<a href=\"#{issue_url}\">issue ##{issue_iid}</a>" message = "#{user_name} #{state} #{issue_link} in #{project_link}: <b>#{title}</b>" - if description - description = format_body(description) - message << description - end + message << "<pre>#{markdown(description)}</pre>" message end @@ -159,23 +177,20 @@ class HipchatService < Service merge_request_id = obj_attr[:iid] state = obj_attr[:state] description = obj_attr[:description] - title = obj_attr[:title] + title = render_line(obj_attr[:title]) merge_request_url = "#{project_url}/merge_requests/#{merge_request_id}" merge_request_link = "<a href=\"#{merge_request_url}\">merge request !#{merge_request_id}</a>" message = "#{user_name} #{state} #{merge_request_link} in " \ "#{project_link}: <b>#{title}</b>" - if description - description = format_body(description) - message << description - end + message << "<pre>#{markdown(description)}</pre>" message end def format_title(title) - "<b>" + title.lines.first.chomp + "</b>" + "<b>#{render_line(title)}</b>" end def create_note_message(data) @@ -186,11 +201,13 @@ class HipchatService < Service note = obj_attr[:note] note_url = obj_attr[:url] noteable_type = obj_attr[:noteable_type] + commit_id = nil case noteable_type when "Commit" commit_attr = HashWithIndifferentAccess.new(data[:commit]) - subject_desc = commit_attr[:id] + commit_id = commit_attr[:id] + subject_desc = commit_id subject_desc = Commit.truncate_sha(subject_desc) subject_type = "commit" title = format_title(commit_attr[:message]) @@ -218,10 +235,7 @@ class HipchatService < Service message = "#{user_name} commented on #{subject_html} in #{project_link}: " message << title - if note - note = format_body(note) - message << note - end + message << "<pre>#{markdown(note, ref: commit_id)}</pre>" message end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index d1df6d0292f..207bb816ad1 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -1,8 +1,12 @@ class IssueTrackerService < Service - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? - default_value_for :category, 'issue_tracker' + # Pattern used to extract links from comments + # Override this method on services that uses different patterns + def reference_pattern + @reference_pattern ||= %r{(\b[A-Z][A-Z0-9_]+-|#{Issue.reference_prefix})(?<issue>\d+)} + end + def default? default end @@ -32,18 +36,24 @@ class IssueTrackerService < Service ] end - def initialize_properties - if properties.nil? - if enabled_in_gitlab_config + # Initialize with default properties values + # or receive a block with custom properties + def initialize_properties(&block) + return unless properties.nil? + + if enabled_in_gitlab_config + if block_given? + yield + else self.properties = { title: issues_tracker['title'], project_url: issues_tracker['project_url'], issues_url: issues_tracker['issues_url'], new_issue_url: issues_tracker['new_issue_url'] } - else - self.properties = {} end + else + self.properties = {} end end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 97bcbacf2b9..70bbbbcda85 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -1,30 +1,65 @@ class JiraService < IssueTrackerService - include HTTParty include Gitlab::Routing.url_helpers - DEFAULT_API_VERSION = 2 + validates :url, url: true, presence: true, if: :activated? + validates :project_key, presence: true, if: :activated? - prop_accessor :username, :password, :api_url, :jira_issue_transition_id, - :title, :description, :project_url, :issues_url, :new_issue_url + prop_accessor :username, :password, :url, :project_key, + :jira_issue_transition_id, :title, :description - validates :api_url, presence: true, url: true, if: :activated? + before_update :reset_password - before_validation :set_api_url, :set_jira_issue_transition_id + def supported_events + %w(commit merge_request) + end - before_update :reset_password + # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 + def reference_pattern + @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} + end + + def initialize_properties + super do + self.properties = { + title: issues_tracker['title'], + url: issues_tracker['url'] + } + end + end def reset_password # don't reset the password if a new one is provided - if api_url_changed? && !password_touched? + if url_changed? && !password_touched? self.password = nil end end + def options + url = URI.parse(self.url) + + { + username: self.username, + password: self.password, + site: URI.join(url, '/').to_s, + context_path: url.path, + auth_type: :basic, + read_timeout: 120, + use_ssl: url.scheme == 'https' + } + end + + def client + @client ||= JIRA::Client.new(options) + end + + def jira_project + @jira_project ||= jira_request { client.Project.find(project_key) } + end + def help - 'Setting `project_url`, `issues_url` and `new_issue_url` will '\ - 'allow a user to easily navigate to the Jira issue tracker. See the '\ - '[integration doc](http://doc.gitlab.com/ce/integration/external-issue-tracker.html) '\ - 'for details.' + 'You need to configure JIRA before enabling this service. For more details + read the + [JIRA service documentation](https://docs.gitlab.com/ce/project_services/jira.html).' end def title @@ -48,12 +83,26 @@ class JiraService < IssueTrackerService end def fields - super.push( - { type: 'text', name: 'api_url', placeholder: 'https://jira.example.com/rest/api/2' }, + [ + { type: 'text', name: 'url', title: 'URL', placeholder: 'https://jira.example.com' }, + { type: 'text', name: 'project_key', placeholder: 'Project Key' }, { type: 'text', name: 'username', placeholder: '' }, { type: 'password', name: 'password', placeholder: '' }, { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' } - ) + ] + end + + # URLs to redirect from Gitlab issues pages to jira issue tracker + def project_url + "#{url}/issues/?jql=project=#{project_key}" + end + + def issues_url + "#{url}/browse/:id" + end + + def new_issue_url + "#{url}/secure/CreateIssue.jspa" end def execute(push, issue = nil) @@ -62,21 +111,26 @@ class JiraService < IssueTrackerService # we just want to test settings test_settings else - close_issue(push, issue) + jira_issue = jira_request { client.Issue.find(issue.iid) } + + return false unless jira_issue.present? + + close_issue(push, jira_issue) end end def create_cross_reference_note(mentioned, noteable, author) - issue_name = mentioned.id - project = self.project - noteable_name = noteable.class.name.underscore.downcase - noteable_id = if noteable.is_a?(Commit) - noteable.id - else - noteable.iid - end + unless can_cross_reference?(noteable) + return "Events for #{noteable.model_name.plural.humanize(capitalize: false)} are disabled." + end + + jira_issue = jira_request { client.Issue.find(mentioned.id) } - entity_url = build_entity_url(noteable_name.to_sym, noteable_id) + return unless jira_issue.present? + + noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id + noteable_type = noteable_name(noteable) + entity_url = build_entity_url(noteable_type, noteable_id) data = { user: { @@ -84,194 +138,173 @@ class JiraService < IssueTrackerService url: resource_url(user_path(author)), }, project: { - name: project.path_with_namespace, - url: resource_url(namespace_project_path(project.namespace, project)) + name: self.project.path_with_namespace, + url: resource_url(namespace_project_path(project.namespace, self.project)) }, entity: { - name: noteable_name.humanize.downcase, + name: noteable_type.humanize.downcase, url: entity_url, title: noteable.title } } - add_comment(data, issue_name) + add_comment(data, jira_issue) end - def test_settings - return unless api_url.present? - result = JiraService.get( - jira_api_test_url, - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) - - case result.code - when 201, 200 - Rails.logger.info("#{self.class.name} SUCCESS #{result.code}: Successfully connected to #{api_url}.") - true - else - Rails.logger.info("#{self.class.name} ERROR #{result.code}: #{result.parsed_response}") - false - end - rescue Errno::ECONNREFUSED => e - Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{api_url}." - false + # reason why service cannot be tested + def disabled_title + "Please fill in Password and Username." end - private + def can_test? + username.present? && password.present? + end - def build_api_url_from_project_url - server = URI(project_url) - default_ports = [["http", 80], ["https", 443]].include?([server.scheme, server.port]) - server_url = "#{server.scheme}://#{server.host}" - server_url.concat(":#{server.port}") unless default_ports - "#{server_url}/rest/api/#{DEFAULT_API_VERSION}" - rescue - "" # looks like project URL was not valid + # JIRA does not need test data. + # We are requesting the project that belongs to the project key. + def test_data(user = nil, project = nil) + nil end - def set_api_url - self.api_url = build_api_url_from_project_url if self.api_url.blank? + def test_settings + return unless url.present? + # Test settings by getting the project + jira_request { jira_project.present? } end - def set_jira_issue_transition_id - self.jira_issue_transition_id ||= "2" + private + + def can_cross_reference?(noteable) + case noteable + when Commit then commit_events + when MergeRequest then merge_requests_events + else true + end end def close_issue(entity, issue) + return if issue.nil? || issue.resolution.present? || !jira_issue_transition_id.present? + commit_id = if entity.is_a?(Commit) entity.id elsif entity.is_a?(MergeRequest) entity.diff_head_sha end + commit_url = build_entity_url(:commit, commit_id) # Depending on the JIRA project's workflow, a comment during transition - # may or may not be allowed. Split the operation in to two calls so the - # comment always works. - transition_issue(issue) - add_issue_solved_comment(issue, commit_id, commit_url) + # may or may not be allowed. Refresh the issue after transition and check + # if it is closed, so we don't have one comment for every commit. + issue = jira_request { client.Issue.find(issue.key) } if transition_issue(issue) + add_issue_solved_comment(issue, commit_id, commit_url) if issue.resolution end def transition_issue(issue) - message = { - transition: { - id: jira_issue_transition_id - } - } - send_message(close_issue_url(issue.iid), message.to_json) + issue.transitions.build.save(transition: { id: jira_issue_transition_id }) end def add_issue_solved_comment(issue, commit_id, commit_url) - comment = { - body: "Issue solved with [#{commit_id}|#{commit_url}]." - } - - send_message(comment_url(issue.iid), comment.to_json) + link_title = "GitLab: Solved by commit #{commit_id}." + comment = "Issue solved with [#{commit_id}|#{commit_url}]." + link_props = build_remote_link_props(url: commit_url, title: link_title, resolved: true) + send_message(issue, comment, link_props) end - def add_comment(data, issue_name) - url = comment_url(issue_name) - user_name = data[:user][:name] - user_url = data[:user][:url] - entity_name = data[:entity][:name] - entity_url = data[:entity][:url] + def add_comment(data, issue) + user_name = data[:user][:name] + user_url = data[:user][:url] + entity_name = data[:entity][:name] + entity_url = data[:entity][:url] entity_title = data[:entity][:title] project_name = data[:project][:name] - message = { - body: %Q{[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'} - } + message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'" + link_title = "GitLab: Mentioned on #{entity_name} - #{entity_title}" + link_props = build_remote_link_props(url: entity_url, title: link_title) - unless existing_comment?(issue_name, message[:body]) - send_message(url, message.to_json) + unless comment_exists?(issue, message) + send_message(issue, message, link_props) end end - def auth - require 'base64' - Base64.urlsafe_encode64("#{self.username}:#{self.password}") + def comment_exists?(issue, message) + comments = jira_request { issue.comments } + + comments.present? && comments.any? { |comment| comment.body.include?(message) } end - def send_message(url, message) - return unless api_url.present? - result = JiraService.post( - url, - body: message, - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) + def send_message(issue, message, remote_link_props) + return unless url.present? - message = case result.code - when 201, 200, 204 - "#{self.class.name} SUCCESS #{result.code}: Successfully posted to #{url}." - when 401 - "#{self.class.name} ERROR 401: Unauthorized. Check the #{self.username} credentials and JIRA access permissions and try again." - else - "#{self.class.name} ERROR #{result.code}: #{result.parsed_response}" - end - - Rails.logger.info(message) - message - rescue URI::InvalidURIError, Errno::ECONNREFUSED => e - Rails.logger.info "#{self.class.name} ERROR: #{e.message}. Hostname: #{url}." - end - - def existing_comment?(issue_name, new_comment) - return unless api_url.present? - result = JiraService.get( - comment_url(issue_name), - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) + jira_request do + if issue.comments.build.save!(body: message) + remote_link = issue.remotelink.build + remote_link.save!(remote_link_props) + result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}." + end - case result.code - when 201, 200 - existing_comments = JSON.parse(result.body)['comments'] + Rails.logger.info(result_message) + result_message + end + end - if existing_comments.present? - return existing_comments.map { |comment| comment['body'].include?(new_comment) }.any? - end + # Build remote link on JIRA properties + # Icons here must be available on WEB so JIRA can read the URL + # We are using a open word graphics icon which have LGPL license + def build_remote_link_props(url:, title:, resolved: false) + status = { + resolved: resolved + } + + if resolved + status[:icon] = { + title: 'Closed', + url16x16: 'http://www.openwebgraphics.com/resources/data/1768/16x16_apply.png' + } end - false - rescue JSON::ParserError - false + { + GlobalID: 'GitLab', + object: { + url: url, + title: title, + status: status, + icon: { title: 'GitLab', url16x16: 'https://gitlab.com/favicon.ico' } + } + } end def resource_url(resource) - "#{Settings.gitlab['url'].chomp("/")}#{resource}" - end - - def build_entity_url(entity_name, entity_id) - resource_url( - polymorphic_url( - [ - self.project.namespace.becomes(Namespace), - self.project, - entity_name - ], - id: entity_id, - routing_type: :path - ) - ) + "#{Settings.gitlab.base_url.chomp("/")}#{resource}" end - def close_issue_url(issue_name) - "#{self.api_url}/issue/#{issue_name}/transitions" + def build_entity_url(noteable_type, entity_id) + polymorphic_url( + [ + self.project.namespace.becomes(Namespace), + self.project, + noteable_type.to_sym + ], + id: entity_id, + host: Settings.gitlab.base_url + ) end - def comment_url(issue_name) - "#{self.api_url}/issue/#{issue_name}/comment" + def noteable_name(noteable) + name = noteable.model_name.singular + + # ProjectSnippet inherits from Snippet class so it causes + # routing error building the URL. + name == "project_snippet" ? "snippet" : name end - def jira_api_test_url - "#{self.api_url}/myself" + # Handle errors when doing JIRA API calls + def jira_request + yield + + rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, URI::InvalidURIError, JIRA::HTTPError => e + Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}" + nil end end diff --git a/app/models/project_services/mattermost_slash_commands_service.rb b/app/models/project_services/mattermost_slash_commands_service.rb new file mode 100644 index 00000000000..33431f41dc2 --- /dev/null +++ b/app/models/project_services/mattermost_slash_commands_service.rb @@ -0,0 +1,49 @@ +class MattermostSlashCommandsService < ChatService + include TriggersHelper + + prop_accessor :token + + def can_test? + false + end + + def title + 'Mattermost Command' + end + + def description + "Perform common operations on GitLab in Mattermost" + end + + def to_param + 'mattermost_slash_commands' + end + + def fields + [ + { type: 'text', name: 'token', placeholder: '' } + ] + end + + def trigger(params) + return nil unless valid_token?(params[:token]) + + user = find_chat_user(params) + unless user + url = authorize_chat_name_url(params) + return Mattermost::Presenter.authorize_chat_name(url) + end + + Gitlab::ChatCommands::Command.new(project, user, params).execute + end + + private + + def find_chat_user(params) + ChatNames::FindUserService.new(self, params).execute + end + + def authorize_chat_name_url(params) + ChatNames::AuthorizeUserService.new(self, params).execute + end +end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb new file mode 100644 index 00000000000..745f9bd1b43 --- /dev/null +++ b/app/models/project_services/pipelines_email_service.rb @@ -0,0 +1,84 @@ +class PipelinesEmailService < Service + prop_accessor :recipients + boolean_accessor :notify_only_broken_pipelines + validates :recipients, presence: true, if: :activated? + + def initialize_properties + self.properties ||= { notify_only_broken_pipelines: true } + end + + def title + 'Pipelines emails' + end + + def description + 'Email the pipelines status to a list of recipients.' + end + + def to_param + 'pipelines_email' + end + + def supported_events + %w[pipeline] + end + + def execute(data, force: false) + return unless supported_events.include?(data[:object_kind]) + return unless force || should_pipeline_be_notified?(data) + + all_recipients = retrieve_recipients(data) + + return unless all_recipients.any? + + pipeline_id = data[:object_attributes][:id] + PipelineNotificationWorker.new.perform(pipeline_id, all_recipients) + end + + def can_test? + project.pipelines.any? + end + + def disabled_title + 'Please setup a pipeline on your repository.' + end + + def test_data(project, user) + data = Gitlab::DataBuilder::Pipeline.build(project.pipelines.last) + data[:user] = user.hook_attrs + data + end + + def fields + [ + { type: 'textarea', + name: 'recipients', + placeholder: 'Emails separated by comma' }, + { type: 'checkbox', + name: 'notify_only_broken_pipelines' }, + ] + end + + def test(data) + result = execute(data, force: true) + + { success: true, result: result } + rescue StandardError => error + { success: false, result: error } + end + + def should_pipeline_be_notified?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end + + def retrieve_recipients(data) + recipients.to_s.split(',').reject(&:blank?) + end +end diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index f634e0772c0..f9da273cf08 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -1,4 +1,6 @@ class RedmineService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title diff --git a/app/models/project_services/slack_service/issue_message.rb b/app/models/project_services/slack_service/issue_message.rb index 88e053ec192..cd87a79d0c6 100644 --- a/app/models/project_services/slack_service/issue_message.rb +++ b/app/models/project_services/slack_service/issue_message.rb @@ -11,7 +11,7 @@ class SlackService attr_reader :description def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_services/slack_service/merge_message.rb b/app/models/project_services/slack_service/merge_message.rb index 11fc691022b..b7615c96068 100644 --- a/app/models/project_services/slack_service/merge_message.rb +++ b/app/models/project_services/slack_service/merge_message.rb @@ -10,7 +10,7 @@ class SlackService attr_reader :title def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_services/slack_service/note_message.rb b/app/models/project_services/slack_service/note_message.rb index 89ba51cb662..797c5937f09 100644 --- a/app/models/project_services/slack_service/note_message.rb +++ b/app/models/project_services/slack_service/note_message.rb @@ -10,7 +10,7 @@ class SlackService def initialize(params) params = HashWithIndifferentAccess.new(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] @@ -46,25 +46,25 @@ class SlackService commit_sha = commit[:id] commit_sha = Commit.truncate_sha(commit_sha) commented_on_message( - "[commit #{commit_sha}](#{@note_url})", + "commit #{commit_sha}", format_title(commit[:message])) end def create_issue_note(issue) commented_on_message( - "[issue ##{issue[:iid]}](#{@note_url})", + "issue ##{issue[:iid]}", format_title(issue[:title])) end def create_merge_note(merge_request) commented_on_message( - "[merge request !#{merge_request[:iid]}](#{@note_url})", + "merge request !#{merge_request[:iid]}", format_title(merge_request[:title])) end def create_snippet_note(snippet) commented_on_message( - "[snippet ##{snippet[:id]}](#{@note_url})", + "snippet ##{snippet[:id]}", format_title(snippet[:title])) end @@ -76,8 +76,8 @@ class SlackService "[#{@project_name}](#{@project_url})" end - def commented_on_message(target_link, title) - @message = "#{@user_name} commented on #{target_link} in #{project_link}: *#{title}*" + def commented_on_message(target, title) + @message = "#{@user_name} [commented on #{target}](#{@note_url}) in #{project_link}: *#{title}*" end end end diff --git a/app/models/project_services/slack_service/pipeline_message.rb b/app/models/project_services/slack_service/pipeline_message.rb index f06b3562965..f8d03c0e2fa 100644 --- a/app/models/project_services/slack_service/pipeline_message.rb +++ b/app/models/project_services/slack_service/pipeline_message.rb @@ -1,11 +1,10 @@ class SlackService class PipelineMessage < BaseMessage - attr_reader :sha, :ref_type, :ref, :status, :project_name, :project_url, + attr_reader :ref_type, :ref, :status, :project_name, :project_url, :user_name, :duration, :pipeline_id def initialize(data) pipeline_attributes = data[:object_attributes] - @sha = pipeline_attributes[:sha] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] @@ -14,7 +13,7 @@ class SlackService @project_name = data[:project][:path_with_namespace] @project_url = data[:project][:web_url] - @user_name = data[:commit] && data[:commit][:author_name] + @user_name = data[:user] && data[:user][:name] end def pretext @@ -73,7 +72,7 @@ class SlackService end def pipeline_link - "[#{Commit.truncate_sha(sha)}](#{pipeline_url})" + "[##{pipeline_id}](#{pipeline_url})" end end end diff --git a/app/models/project_services/slack_service/wiki_page_message.rb b/app/models/project_services/slack_service/wiki_page_message.rb index f336d9e7691..160ca3ac115 100644 --- a/app/models/project_services/slack_service/wiki_page_message.rb +++ b/app/models/project_services/slack_service/wiki_page_message.rb @@ -9,7 +9,7 @@ class SlackService attr_reader :description def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_team.rb b/app/models/project_team.rb index ab6ea2aae36..8a53e974b6f 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -21,6 +21,22 @@ class ProjectTeam end end + def add_guest(user, current_user: nil) + self << [user, :guest, current_user] + end + + def add_reporter(user, current_user: nil) + self << [user, :reporter, current_user] + end + + def add_developer(user, current_user: nil) + self << [user, :developer, current_user] + end + + def add_master(user, current_user: nil) + self << [user, :master, current_user] + end + def find_member(user_id) member = project.members.find_by(user_id: user_id) @@ -33,18 +49,24 @@ class ProjectTeam member end - def add_users(users, access, current_user: nil, expires_at: nil) + def add_users(users, access_level, current_user: nil, expires_at: nil) ProjectMember.add_users_to_projects( [project.id], users, - access, + access_level, current_user: current_user, expires_at: expires_at ) end - def add_user(user, access, current_user: nil, expires_at: nil) - add_users([user], access, current_user: current_user, expires_at: expires_at) + def add_user(user, access_level, current_user: nil, expires_at: nil) + ProjectMember.add_user( + project, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) end # Remove all users from project team @@ -58,19 +80,19 @@ class ProjectTeam alias_method :users, :members def guests - @guests ||= fetch_members(:guests) + @guests ||= fetch_members(Gitlab::Access::GUEST) end def reporters - @reporters ||= fetch_members(:reporters) + @reporters ||= fetch_members(Gitlab::Access::REPORTER) end def developers - @developers ||= fetch_members(:developers) + @developers ||= fetch_members(Gitlab::Access::DEVELOPER) end def masters - @masters ||= fetch_members(:masters) + @masters ||= fetch_members(Gitlab::Access::MASTER) end def import(source_project, current_user = nil) @@ -119,14 +141,12 @@ class ProjectTeam max_member_access(user.id) == Gitlab::Access::MASTER end - def member?(user, min_member_access = nil) - member = !!find_member(user.id) + # Checks if `user` is authorized for this project, with at least the + # `min_access_level` (if given). + def member?(user, min_access_level = Gitlab::Access::GUEST) + return false unless user - if min_member_access - member && max_member_access(user.id) >= min_member_access - else - member - end + user.authorized_project?(project, min_access_level) end def human_max_access(user_id) @@ -149,104 +169,29 @@ class ProjectTeam # Lookup only the IDs we need user_ids = user_ids - access.keys + users_access = project.project_authorizations. + where(user: user_ids). + group(:user_id). + maximum(:access_level) - if user_ids.present? - user_ids.each { |id| access[id] = Gitlab::Access::NO_ACCESS } - - member_access = project.members.access_for_user_ids(user_ids) - merge_max!(access, member_access) - - if group - group_access = group.members.access_for_user_ids(user_ids) - merge_max!(access, group_access) - end - - # Each group produces a list of maximum access level per user. We take the - # max of the values produced by each group. - if project.invited_groups.any? && project.allowed_to_share_with_group? - project.project_group_links.each do |group_link| - invited_access = max_invited_level_for_users(group_link, user_ids) - merge_max!(access, invited_access) - end - end - end - + access.merge!(users_access) access end def max_member_access(user_id) - max_member_access_for_user_ids([user_id])[user_id] + max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS end private - # For a given group, return the maximum access level for the user. This is the min of - # the invited access level of the group and the access level of the user within the group. - # For example, if the group has been given DEVELOPER access but the member has MASTER access, - # the user should receive only DEVELOPER access. - def max_invited_level_for_users(group_link, user_ids) - invited_group = group_link.group - capped_access_level = group_link.group_access - access = invited_group.group_members.access_for_user_ids(user_ids) - - # If the user is not in the list, assume he/she does not have access - missing_users = user_ids - access.keys - missing_users.each { |id| access[id] = Gitlab::Access::NO_ACCESS } - - # Cap the maximum access by the invited level access - access.each { |key, value| access[key] = [value, capped_access_level].min } - end - def fetch_members(level = nil) - project_members = project.members - group_members = group ? group.members : [] - invited_members = [] - - if project.invited_groups.any? && project.allowed_to_share_with_group? - project.project_group_links.includes(group: [:group_members]).each do |group_link| - invited_group = group_link.group - im = invited_group.members - - if level - int_level = GroupMember.access_level_roles[level.to_s.singularize.titleize] - - # Skip group members if we ask for masters - # but max group access is developers - next if int_level > group_link.group_access - - # If we ask for developers and max - # group access is developers we need to provide - # both group master, developers as devs - if int_level == group_link.group_access - im.where("access_level >= ?)", group_link.group_access) - else - im.send(level) - end - end - - invited_members << im - end - - invited_members = invited_members.flatten.compact - end - - if level - project_members = project_members.send(level) - group_members = group_members.send(level) if group - end - - user_ids = project_members.pluck(:user_id) - user_ids.push(*invited_members.map(&:user_id)) if invited_members.any? - user_ids.push(*group_members.pluck(:user_id)) if group + members = project.authorized_users + members = members.where(project_authorizations: { access_level: level }) if level - User.where(id: user_ids) + members end def group project.group end - - def merge_max!(first_hash, second_hash) - first_hash.merge!(second_hash) { |_key, old, new| old > new ? old : new } - end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 46f70da2452..9db96347322 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -127,7 +127,7 @@ class ProjectWiki end def search_files(query) - repository.search_files(query, default_branch) + repository.search_files_by_content(query, default_branch) end def repository diff --git a/app/models/release.rb b/app/models/release.rb index e196b84eb18..c936899799e 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -1,4 +1,8 @@ class Release < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :description + belongs_to :project validates :description, :project, :tag, presence: true diff --git a/app/models/repository.rb b/app/models/repository.rb index c69e5a22a69..bf136ccdb6c 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1,15 +1,57 @@ require 'securerandom' class Repository + include Gitlab::ShellAdapter + + attr_accessor :path_with_namespace, :project + class CommitError < StandardError; end - # Files to use as a project avatar in case no avatar was uploaded via the web - # UI. - AVATAR_FILES = %w{logo.png logo.jpg logo.gif} + # Methods that cache data from the Git repository. + # + # Each entry in this Array should have a corresponding method with the exact + # same name. The cache key used by those methods must also match method's + # name. + # + # For example, for entry `:readme` there's a method called `readme` which + # stores its data in the `readme` cache key. + CACHED_METHODS = %i(size commit_count readme version contribution_guide + changelog license_blob license_key gitignore koding_yml + gitlab_ci_yml branch_names tag_names branch_count + tag_count avatar exists? empty? root_ref) + + # Certain method caches should be refreshed when certain types of files are + # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to + # the corresponding methods to call for refreshing caches. + METHOD_CACHES_FOR_FILE_TYPES = { + readme: :readme, + changelog: :changelog, + license: %i(license_blob license_key), + contributing: :contribution_guide, + version: :version, + gitignore: :gitignore, + koding: :koding_yml, + gitlab_ci: :gitlab_ci_yml, + avatar: :avatar + } + + # Wraps around the given method and caches its output in Redis and an instance + # variable. + # + # This only works for methods that do not take any arguments. + def self.cache_method(name, fallback: nil) + original = :"_uncached_#{name}" - include Gitlab::ShellAdapter + alias_method(original, name) - attr_accessor :path_with_namespace, :project + define_method(name) do + cache_method_output(name, fallback: fallback) { __send__(original) } + end + end + + def self.storages + Gitlab.config.repositories.storages + end def initialize(path_with_namespace, project) @path_with_namespace = path_with_namespace @@ -33,24 +75,6 @@ class Repository ) end - def exists? - return @exists unless @exists.nil? - - @exists = cache.fetch(:exists?) do - begin - raw_repository && raw_repository.rugged ? true : false - rescue Gitlab::Git::Repository::NoRepository - false - end - end - end - - def empty? - return @empty unless @empty.nil? - - @empty = cache.fetch(:empty?) { raw_repository.empty? } - end - # # Git repository can contains some hidden refs like: # /refs/notes/* @@ -70,15 +94,17 @@ class Repository def commit(ref = 'HEAD') return nil unless exists? + commit = if ref.is_a?(Gitlab::Git::Commit) ref else Gitlab::Git::Commit.find(raw_repository, ref) end + commit = ::Commit.new(commit, @project) if commit commit - rescue Rugged::OdbError + rescue Rugged::OdbError, Rugged::TreeError nil end @@ -109,15 +135,20 @@ class Repository end def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0) + unless exists? && has_visible_content? && query.present? + return [] + end + ref ||= root_ref - # Limited to 1000 commits for now, could be parameterized? - args = %W(#{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} --max-count #{limit} --grep=#{query}) + args = %W( + #{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} + --max-count #{limit} --grep=#{query} --regexp-ignore-case + ) args = args.concat(%W(-- #{path})) if path.present? - git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:chomp) - commits = git_log_results.map { |c| commit(c) } - commits + git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines + git_log_results.map { |c| commit(c.chomp) }.compact end def find_branch(name, fresh_repo: true) @@ -165,18 +196,25 @@ class Repository options = { message: message, tagger: user_to_committer(user) } if message - GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do - rugged.tags.create(tag_name, target, options) + rugged.tags.create(tag_name, target, options) + tag = find_tag(tag_name) + + GitHooksService.new.execute(user, path_to_repo, oldrev, tag.target, ref) do + # we already created a tag, because we need tag SHA to pass correct + # values to hooks end - find_tag(tag_name) + tag + rescue GitHooksService::PreReceiveError + rugged.tags.delete(tag_name) + raise end def rm_branch(user, branch_name) before_remove_branch branch = find_branch(branch_name) - oldrev = branch.try(:target).try(:id) + oldrev = branch.try(:dereferenced_target).try(:id) newrev = Gitlab::Git::BLANK_SHA ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name @@ -203,16 +241,14 @@ class Repository branch_names + tag_names end - def branch_names - @branch_names ||= cache.fetch(:branch_names) { branches.map(&:name) } - end - def branch_exists?(branch_name) branch_names.include?(branch_name) end def ref_exists?(ref) rugged.references.exist?(ref) + rescue Rugged::ReferenceError + false end def update_ref!(name, newrev, oldrev) @@ -220,7 +256,7 @@ class Repository # offer 'compare and swap' ref updates. Without compare-and-swap we can # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %w[git update-ref --stdin -z] + command = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z) _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin| stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00") end @@ -251,39 +287,7 @@ class Repository end def kept_around?(sha) - begin - ref_exists?(keep_around_ref_name(sha)) - rescue Rugged::ReferenceError - false - end - end - - def tag_names - cache.fetch(:tag_names) { raw_repository.tag_names } - end - - def commit_count - cache.fetch(:commit_count) do - begin - raw_repository.commit_count(self.root_ref) - rescue - 0 - end - end - end - - def branch_count - @branch_count ||= cache.fetch(:branch_count) { branches.size } - end - - def tag_count - @tag_count ||= cache.fetch(:tag_count) { raw_repository.rugged.tags.count } - end - - # Return repo size in megabytes - # Cached in redis - def size - cache.fetch(:size) { raw_repository.size } + ref_exists?(keep_around_ref_name(sha)) end def diverging_commit_counts(branch) @@ -292,57 +296,64 @@ class Repository # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes number_commits_behind = raw_repository. - count_commits_between(branch.target.sha, root_ref_hash) + count_commits_between(branch.dereferenced_target.sha, root_ref_hash) number_commits_ahead = raw_repository. - count_commits_between(root_ref_hash, branch.target.sha) + count_commits_between(root_ref_hash, branch.dereferenced_target.sha) { behind: number_commits_behind, ahead: number_commits_ahead } end end - # Keys for data that can be affected for any commit push. - def cache_keys - %i(size commit_count - readme version contribution_guide changelog - license_blob license_key gitignore koding_yml) + def expire_tags_cache + expire_method_caches(%i(tag_names tag_count)) + @tags = nil end - # Keys for data on branch/tag operations. - def cache_keys_for_branches_and_tags - %i(branch_names tag_names branch_count tag_count) + def expire_branches_cache + expire_method_caches(%i(branch_names branch_count)) + @local_branches = nil end - def build_cache - (cache_keys + cache_keys_for_branches_and_tags).each do |key| - unless cache.exist?(key) - send(key) - end - end + def expire_statistics_caches + expire_method_caches(%i(size commit_count)) end - def expire_tags_cache - cache.expire(:tag_names) - @tags = nil + def expire_all_method_caches + expire_method_caches(CACHED_METHODS) end - def expire_branches_cache - cache.expire(:branch_names) - @branch_names = nil - @local_branches = nil + # Expires the caches of a specific set of methods + def expire_method_caches(methods) + methods.each do |key| + cache.expire(key) + + ivar = cache_instance_variable_name(key) + + remove_instance_variable(ivar) if instance_variable_defined?(ivar) + end end - def expire_cache(branch_name = nil, revision = nil) - cache_keys.each do |key| - cache.expire(key) + def expire_avatar_cache + expire_method_caches(%i(avatar)) + end + + # Refreshes the method caches of this repository. + # + # types - An Array of file types (e.g. `:readme`) used to refresh extra + # caches. + def refresh_method_caches(types) + to_refresh = [] + + types.each do |type| + methods = METHOD_CACHES_FOR_FILE_TYPES[type.to_sym] + + to_refresh.concat(Array(methods)) if methods end - expire_branch_cache(branch_name) - expire_avatar_cache(branch_name, revision) + expire_method_caches(to_refresh) - # This ensures this particular cache is flushed after the first commit to a - # new repository. - expire_emptiness_caches if empty? + to_refresh.each { |method| send(method) } end def expire_branch_cache(branch_name = nil) @@ -361,15 +372,14 @@ class Repository end def expire_root_ref_cache - cache.expire(:root_ref) - @root_ref = nil + expire_method_caches(%i(root_ref)) end # Expires the cache(s) used to determine if a repository is empty or not. def expire_emptiness_caches - cache.expire(:empty?) - @empty = nil + return unless empty? + expire_method_caches(%i(empty?)) expire_has_visible_content_cache end @@ -378,40 +388,22 @@ class Repository @has_visible_content = nil end - def expire_branch_count_cache - cache.expire(:branch_count) - @branch_count = nil - end - - def expire_tag_count_cache - cache.expire(:tag_count) - @tag_count = nil - end - def lookup_cache @lookup_cache ||= {} end - def expire_avatar_cache(branch_name = nil, revision = nil) - # Avatars are pulled from the default branch, thus if somebody pushes to a - # different branch there's no need to expire anything. - return if branch_name && branch_name != root_ref - - # We don't want to flush the cache if the commit didn't actually make any - # changes to any of the possible avatar files. - if revision && commit = self.commit(revision) - return unless commit.raw_diffs(deltas_only: true). - any? { |diff| AVATAR_FILES.include?(diff.new_path) } - end - - cache.expire(:avatar) - - @avatar = nil + def expire_exists_cache + expire_method_caches(%i(exists?)) end - def expire_exists_cache - cache.expire(:exists?) - @exists = nil + # expire cache that doesn't depend on repository data (when expiring) + def expire_content_cache + expire_tags_cache + expire_branches_cache + expire_root_ref_cache + expire_emptiness_caches + expire_exists_cache + expire_statistics_caches end # Runs code after a repository has been created. @@ -426,17 +418,9 @@ class Repository # Runs code just before a repository is deleted. def before_delete expire_exists_cache - - expire_cache if exists? - - # expire cache that don't depend on repository data (when expiring) - expire_tags_cache - expire_tag_count_cache - expire_branches_cache - expire_branch_count_cache - expire_root_ref_cache - expire_emptiness_caches - expire_exists_cache + expire_all_method_caches + expire_branch_cache if exists? + expire_content_cache repository_event(:remove_repository) end @@ -452,9 +436,9 @@ class Repository # Runs code before pushing (= creating or removing) a tag. def before_push_tag - expire_cache + expire_statistics_caches + expire_emptiness_caches expire_tags_cache - expire_tag_count_cache repository_event(:push_tag) end @@ -462,25 +446,26 @@ class Repository # Runs code before removing a tag. def before_remove_tag expire_tags_cache - expire_tag_count_cache + expire_statistics_caches repository_event(:remove_tag) end def before_import - expire_emptiness_caches - expire_exists_cache + expire_content_cache end # Runs code after a repository has been forked/imported. def after_import - expire_emptiness_caches - expire_exists_cache + expire_content_cache + expire_tags_cache + expire_branches_cache end # Runs code after a new commit has been pushed. - def after_push_commit(branch_name, revision) - expire_cache(branch_name, revision) + def after_push_commit(branch_name) + expire_statistics_caches + expire_branch_cache(branch_name) repository_event(:push_commit, branch: branch_name) end @@ -489,7 +474,6 @@ class Repository def after_create_branch expire_branches_cache expire_has_visible_content_cache - expire_branch_count_cache repository_event(:push_branch) end @@ -504,7 +488,6 @@ class Repository # Runs code after an existing branch has been removed. def after_remove_branch expire_has_visible_content_cache - expire_branch_count_cache expire_branches_cache end @@ -531,86 +514,127 @@ class Repository Gitlab::Git::Blob.raw(self, oid) end + def root_ref + if raw_repository + raw_repository.root_ref + else + # When the repo does not exist we raise this error so no data is cached. + raise Rugged::ReferenceError + end + end + cache_method :root_ref + + def exists? + refs_directory_exists? + end + cache_method :exists? + + def empty? + raw_repository.empty? + end + cache_method :empty? + + # The size of this repository in megabytes. + def size + exists? ? raw_repository.size : 0.0 + end + cache_method :size, fallback: 0.0 + + def commit_count + root_ref ? raw_repository.commit_count(root_ref) : 0 + end + cache_method :commit_count, fallback: 0 + + def branch_names + branches.map(&:name) + end + cache_method :branch_names, fallback: [] + + def tag_names + raw_repository.tag_names + end + cache_method :tag_names, fallback: [] + + def branch_count + branches.size + end + cache_method :branch_count, fallback: 0 + + def tag_count + raw_repository.rugged.tags.count + end + cache_method :tag_count, fallback: 0 + + def avatar + if tree = file_on_head(:avatar) + tree.path + end + end + cache_method :avatar + def readme - cache.fetch(:readme) { tree(:head).readme } + if head = tree(:head) + head.readme + end end + cache_method :readme def version - cache.fetch(:version) do - tree(:head).blobs.find do |file| - file.name.casecmp('version').zero? - end - end + file_on_head(:version) end + cache_method :version def contribution_guide - cache.fetch(:contribution_guide) do - tree(:head).blobs.find do |file| - file.contributing? - end - end + file_on_head(:contributing) end + cache_method :contribution_guide def changelog - cache.fetch(:changelog) do - file_on_head(/\A(changelog|history|changes|news)/i) - end + file_on_head(:changelog) end + cache_method :changelog def license_blob - return nil unless head_exists? - - cache.fetch(:license_blob) do - file_on_head(/\A(licen[sc]e|copying)(\..+|\z)/i) - end + file_on_head(:license) end + cache_method :license_blob def license_key - return nil unless head_exists? + return unless exists? - cache.fetch(:license_key) do - Licensee.license(path).try(:key) - end + Licensee.license(path).try(:key) end + cache_method :license_key def gitignore - return nil if !exists? || empty? - - cache.fetch(:gitignore) do - file_on_head(/\A\.gitignore\z/) - end + file_on_head(:gitignore) end + cache_method :gitignore def koding_yml - return nil unless head_exists? - - cache.fetch(:koding_yml) do - file_on_head(/\A\.koding\.yml\z/) - end + file_on_head(:koding) end + cache_method :koding_yml def gitlab_ci_yml - return nil unless head_exists? - - @gitlab_ci_yml ||= tree(:head).blobs.find do |file| - file.name == '.gitlab-ci.yml' - end - rescue Rugged::ReferenceError - # For unknow reason spinach scenario "Scenario: I change project path" - # lead to "Reference 'HEAD' not found" exception from Repository#empty? - nil + file_on_head(:gitlab_ci) end + cache_method :gitlab_ci_yml def head_commit @head_commit ||= commit(self.root_ref) end def head_tree - @head_tree ||= Tree.new(self, head_commit.sha, nil) + if head_commit + @head_tree ||= Tree.new(self, head_commit.sha, nil) + end end - def tree(sha = :head, path = nil) + def tree(sha = :head, path = nil, recursive: false) if sha == :head + return unless head_commit + if path.nil? return head_tree else @@ -618,7 +642,7 @@ class Repository end end - Tree.new(self, sha, path) + Tree.new(self, sha, path, recursive: recursive) end def blob_at_branch(branch_name, path) @@ -674,11 +698,11 @@ class Repository branches.sort_by(&:name) when 'updated_desc' branches.sort do |a, b| - commit(b.target).committed_date <=> commit(a.target).committed_date + commit(b.dereferenced_target).committed_date <=> commit(a.dereferenced_target).committed_date end when 'updated_asc' branches.sort do |a, b| - commit(a.target).committed_date <=> commit(b.target).committed_date + commit(a.dereferenced_target).committed_date <=> commit(b.dereferenced_target).committed_date end else branches @@ -717,6 +741,14 @@ class Repository end end + def ref_name_for_sha(ref_path, sha) + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, path_to_repo).first.split.last + end + def refs_contains_sha(ref_type, sha) args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha}) names = Gitlab::Popen.popen(args, path_to_repo).first @@ -752,66 +784,59 @@ class Repository @tags ||= raw_repository.tags end - def root_ref - @root_ref ||= cache.fetch(:root_ref) { raw_repository.root_ref } - end - - def commit_dir(user, path, message, branch) + def commit_dir(user, path, message, branch, author_email: nil, author_name: nil) update_branch_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + options = { + commit: { + branch: ref, + message: message, + update_ref: false + } } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) + raw_repository.mkdir(path, options) end end - def commit_file(user, path, content, message, branch, update) + def commit_file(user, path, content, message, branch, update, author_email: nil, author_name: nil) update_branch_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + content: content, + path: path, + update: update + } } - options[:file] = { - content: content, - path: path, - update: update - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) Gitlab::Git::Blob.commit(raw_repository, options) end end - def update_file(user, path, content, branch:, previous_path:, message:) + def update_file(user, path, content, branch:, previous_path:, message:, author_email: nil, author_name: nil) update_branch_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + content: content, + path: path, + update: true + } } - options[:file] = { - content: content, - path: path, - update: true - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) if previous_path && previous_path != path options[:file][:previous_path] = previous_path @@ -822,34 +847,85 @@ class Repository end end - def remove_file(user, path, message, branch) + def remove_file(user, path, message, branch, author_email: nil, author_name: nil) update_branch_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + path: path + } } - options[:file] = { - path: path - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) Gitlab::Git::Blob.remove(raw_repository, options) end end - def user_to_committer(user) + def multi_action(user:, branch:, message:, actions:, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + index = rugged.index + parents = [] + branch = find_branch(ref) + + if branch + last_commit = branch.dereferenced_target + index.read_tree(last_commit.raw_commit.tree) + parents = [last_commit.sha] + end + + actions.each do |action| + case action[:action] + when :create, :update, :move + mode = + case action[:action] + when :update + index.get(action[:file_path])[:mode] + when :move + index.get(action[:previous_path])[:mode] + end + mode ||= 0o100644 + + index.remove(action[:previous_path]) if action[:action] == :move + + content = action[:encoding] == 'base64' ? Base64.decode64(action[:content]) : action[:content] + oid = rugged.write(content, :blob) + + index.add(path: action[:file_path], oid: oid, mode: mode) + when :delete + index.remove(action[:file_path]) + end + end + + options = { + tree: index.write_tree(rugged), + message: message, + parents: parents + } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) + + Rugged::Commit.create(rugged, options) + end + end + + def get_committer_and_author(user, email: nil, name: nil) + committer = user_to_committer(user) + author = Gitlab::Git::committer_hash(email: email, name: name) || committer + { - email: user.email, - name: user.name, - time: Time.now + author: author, + committer: committer } end + def user_to_committer(user) + Gitlab::Git::committer_hash(email: user.email, name: user.name) + end + def can_be_merged?(source_sha, target_branch) our_commit = rugged.branches[target_branch].target their_commit = rugged.lookup(source_sha) @@ -884,7 +960,7 @@ class Repository end def revert(user, commit, base_branch, revert_tree_id = nil) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha revert_tree_id ||= check_revert_content(commit, base_branch) return false unless revert_tree_id @@ -901,7 +977,7 @@ class Repository end def cherry_pick(user, commit, base_branch, cherry_pick_tree_id = nil) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha cherry_pick_tree_id ||= check_cherry_pick_content(commit, base_branch) return false unless cherry_pick_tree_id @@ -930,7 +1006,7 @@ class Repository end def check_revert_content(commit, base_branch) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha args = [commit.id, source_sha] args << { mainline: 1 } if commit.merge_commit? @@ -944,7 +1020,7 @@ class Repository end def check_cherry_pick_content(commit, base_branch) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha args = [commit.id, source_sha] args << 1 if commit.merge_commit? @@ -966,7 +1042,8 @@ class Repository root_ref_commit = commit(root_ref) if branch_commit - is_ancestor?(branch_commit.id, root_ref_commit.id) + same_head = branch_commit.id == root_ref_commit.id + !same_head && is_ancestor?(branch_commit.id, root_ref_commit.id) else nil end @@ -984,17 +1061,34 @@ class Repository merge_base(ancestor_id, descendant_id) == ancestor_id end - def search_files(query, ref) + def empty_repo? + !exists? || !has_visible_content? + end + + def search_files_by_content(query, ref) + return [] if empty_repo? || query.blank? + offset = 2 args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/) end + def search_files_by_name(query, ref) + return [] if empty_repo? || query.blank? + + args = %W(#{Gitlab.config.git.bin_path} ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)}) + Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:strip) + end + def fetch_ref(source_path, source_ref, target_ref) args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) Gitlab::Popen.popen(args, path_to_repo) end + def create_ref(ref, ref_path) + fetch_ref(path_to_repo, ref, ref_path) + end + def update_branch_with_hooks(current_user, branch) update_autocrlf_option @@ -1012,7 +1106,7 @@ class Repository if rugged.lookup(newrev).parent_ids.empty? || target_branch.nil? oldrev = Gitlab::Git::BLANK_SHA else - oldrev = rugged.merge_base(newrev, target_branch.target.sha) + oldrev = rugged.merge_base(newrev, target_branch.dereferenced_target.sha) end GitHooksService.new.execute(current_user, path_to_repo, oldrev, newrev, ref) do @@ -1047,32 +1141,59 @@ class Repository end end - def avatar - return nil unless exists? + # Caches the supplied block both in a cache and in an instance variable. + # + # The cache key and instance variable are named the same way as the value of + # the `key` argument. + # + # This method will return `nil` if the corresponding instance variable is also + # set to `nil`. This ensures we don't keep yielding the block when it returns + # `nil`. + # + # key - The name of the key to cache the data in. + # fallback - A value to fall back to in the event of a Git error. + def cache_method_output(key, fallback: nil, &block) + ivar = cache_instance_variable_name(key) - @avatar ||= cache.fetch(:avatar) do - AVATAR_FILES.find do |file| - blob_at_branch(root_ref, file) + if instance_variable_defined?(ivar) + instance_variable_get(ivar) + else + begin + instance_variable_set(ivar, cache.fetch(key, &block)) + rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository + # if e.g. HEAD or the entire repository doesn't exist we want to + # gracefully handle this and not cache anything. + fallback end end end - private + def cache_instance_variable_name(key) + :"@#{key.to_s.tr('?!', '')}" + end - def cache - @cache ||= RepositoryCache.new(path_with_namespace, @project.id) + def file_on_head(type) + if head = tree(:head) + head.blobs.find do |file| + Gitlab::FileDetector.type_of(file.name) == type + end + end end - def head_exists? - exists? && !empty? && !rugged.head_unborn? + private + + def refs_directory_exists? + return false unless path_with_namespace + + File.exist?(File.join(path_to_repo, 'refs')) end - def file_on_head(regex) - tree(:head).blobs.find { |file| file.name =~ regex } + def cache + @cache ||= RepositoryCache.new(path_with_namespace, @project.id) end def tags_sorted_by_committed_date - tags.sort_by { |tag| tag.target.committed_date } + tags.sort_by { |tag| tag.dereferenced_target.committed_date } end def keep_around_ref_name(sha) diff --git a/app/models/service.rb b/app/models/service.rb index 80de7175565..0c36acfc1b7 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -8,6 +8,7 @@ class Service < ActiveRecord::Base default_value_for :push_events, true default_value_for :issues_events, true default_value_for :confidential_issues_events, true + default_value_for :commit_events, true default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true @@ -136,6 +137,7 @@ class Service < ActiveRecord::Base end def #{arg}=(value) + self.properties ||= {} updated_properties['#{arg}'] = #{arg} unless #{arg}_changed? self.properties['#{arg}'] = value end @@ -195,7 +197,7 @@ class Service < ActiveRecord::Base end def self.available_services_names - %w( + %w[ asana assembla bamboo @@ -212,19 +214,21 @@ class Service < ActiveRecord::Base hipchat irker jira + mattermost_slash_commands + pipelines_email pivotaltracker pushover redmine slack teamcity - ) + ] end - def self.create_from_template(project_id, template) + def self.build_from_template(project_id, template) service = template.dup service.template = false service.project_id = project_id - service if service.save + service end private diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 5ec933601ac..8ff4e7ae718 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -1,9 +1,21 @@ class Snippet < ActiveRecord::Base include Gitlab::VisibilityLevel include Linguist::BlobHelper + include CacheMarkdownField include Participable include Referable include Sortable + include Awardable + include Mentionable + + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :content + + # If file_name changes, it invalidates content + alias_method :default_content_html_invalidator, :content_html_invalidated? + def content_html_invalidated? + default_content_html_invalidator || file_name_changed? + end default_value_for :visibility_level, Snippet::PRIVATE diff --git a/app/models/subscription.rb b/app/models/subscription.rb index 3b8aa1eb866..17869c8bac2 100644 --- a/app/models/subscription.rb +++ b/app/models/subscription.rb @@ -1,8 +1,9 @@ class Subscription < ActiveRecord::Base belongs_to :user + belongs_to :project belongs_to :subscribable, polymorphic: true - validates :user_id, - uniqueness: { scope: [:subscribable_id, :subscribable_type] }, - presence: true + validates :user, :subscribable, presence: true + + validates :project_id, uniqueness: { scope: [:subscribable_id, :subscribable_type, :user_id] } end diff --git a/app/models/todo.rb b/app/models/todo.rb index 6ae9956ade5..f5ade1cc293 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -52,7 +52,13 @@ class Todo < ActiveRecord::Base # Todos with highest priority first then oldest todos # Need to order by created_at last because of differences on Mysql and Postgres when joining by type "Merge_request/Issue" def order_by_labels_priority - highest_priority = highest_label_priority(["Issue", "MergeRequest"], "todos.target_id").to_sql + params = { + target_type_column: "todos.target_type", + target_column: "todos.target_id", + project_column: "todos.project_id" + } + + highest_priority = highest_label_priority(params).to_sql select("#{table_name}.*, (#{highest_priority}) AS highest_priority"). order(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')). diff --git a/app/models/tree.rb b/app/models/tree.rb index 7c4ed6e393b..fe148b0ec65 100644 --- a/app/models/tree.rb +++ b/app/models/tree.rb @@ -3,21 +3,24 @@ class Tree attr_accessor :repository, :sha, :path, :entries - def initialize(repository, sha, path = '/') + def initialize(repository, sha, path = '/', recursive: false) path = '/' if path.blank? @repository = repository @sha = sha @path = path + @recursive = recursive git_repo = @repository.raw_repository - @entries = Gitlab::Git::Tree.where(git_repo, @sha, @path) + @entries = get_entries(git_repo, @sha, @path, recursive: @recursive) end def readme return @readme if defined?(@readme) - available_readmes = blobs.select(&:readme?) + available_readmes = blobs.select do |blob| + Gitlab::FileDetector.type_of(blob.name) == :readme + end previewable_readmes = available_readmes.select do |blob| previewable?(blob.name) @@ -58,4 +61,21 @@ class Tree def sorted_entries trees + blobs + submodules end + + private + + def get_entries(git_repo, sha, path, recursive: false) + current_path_entries = Gitlab::Git::Tree.where(git_repo, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(get_entries(git_repo, sha, entry.path, recursive: true)) + end + end + + ordered_entries + end end diff --git a/app/models/trending_project.rb b/app/models/trending_project.rb new file mode 100644 index 00000000000..27e3732da17 --- /dev/null +++ b/app/models/trending_project.rb @@ -0,0 +1,35 @@ +class TrendingProject < ActiveRecord::Base + belongs_to :project + + # The number of months to include in the trending calculation. + MONTHS_TO_INCLUDE = 1 + + # The maximum number of projects to include in the trending set. + PROJECTS_LIMIT = 100 + + # Populates the trending projects table with the current list of trending + # projects. + def self.refresh! + # The calculation **must** run in a transaction. If the removal of data and + # insertion of new data were to run separately a user might end up with an + # empty list of trending projects for a short period of time. + transaction do + delete_all + + timestamp = connection.quote(MONTHS_TO_INCLUDE.months.ago) + + connection.execute <<-EOF.strip_heredoc + INSERT INTO #{table_name} (project_id) + SELECT project_id + FROM notes + INNER JOIN projects ON projects.id = notes.project_id + WHERE notes.created_at >= #{timestamp} + AND notes.system IS FALSE + AND projects.visibility_level = #{Gitlab::VisibilityLevel::PUBLIC} + GROUP BY project_id + ORDER BY count(*) DESC + LIMIT #{PROJECTS_LIMIT}; + EOF + end + end +end diff --git a/app/models/user.rb b/app/models/user.rb index 6996740eebd..513a19d81d2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -13,6 +13,7 @@ class User < ActiveRecord::Base DEFAULT_NOTIFICATION_LEVEL = :participating add_authentication_token_field :authentication_token + add_authentication_token_field :incoming_email_token default_value_for :admin, false default_value_for(:external) { current_application_settings.user_default_external } @@ -47,7 +48,7 @@ class User < ActiveRecord::Base # # Namespace for personal projects - has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id, class_name: "Namespace" + has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id # Profile has_many :keys, dependent: :destroy @@ -55,6 +56,7 @@ class User < ActiveRecord::Base has_many :personal_access_tokens, dependent: :destroy has_many :identities, dependent: :destroy, autosave: true has_many :u2f_registrations, dependent: :destroy + has_many :chat_names, dependent: :destroy # Groups has_many :members, dependent: :destroy @@ -66,17 +68,19 @@ class User < ActiveRecord::Base # Projects has_many :groups_projects, through: :groups, source: :projects has_many :personal_projects, through: :namespace, source: :projects - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, class_name: 'ProjectMember' + has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy has_many :projects, through: :project_members has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy has_many :starred_projects, through: :users_star_projects, source: :project + has_many :project_authorizations, dependent: :destroy + has_many :authorized_projects, through: :project_authorizations, source: :project - has_many :snippets, dependent: :destroy, foreign_key: :author_id, class_name: "Snippet" + has_many :snippets, dependent: :destroy, foreign_key: :author_id has_many :issues, dependent: :destroy, foreign_key: :author_id has_many :notes, dependent: :destroy, foreign_key: :author_id has_many :merge_requests, dependent: :destroy, foreign_key: :author_id - has_many :events, dependent: :destroy, foreign_key: :author_id, class_name: "Event" + has_many :events, dependent: :destroy, foreign_key: :author_id has_many :subscriptions, dependent: :destroy has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" has_many :assigned_issues, dependent: :destroy, foreign_key: :assignee_id, class_name: "Issue" @@ -93,8 +97,10 @@ class User < ActiveRecord::Base # # Validations # + # Note: devise :validatable above adds validations for :email and :password validates :name, presence: true - validates :notification_email, presence: true, email: true + validates :notification_email, presence: true + validates :notification_email, email: true, if: ->(user) { user.notification_email != user.email } validates :public_email, presence: true, uniqueness: true, email: true, allow_blank: true validates :bio, length: { maximum: 255 }, allow_blank: true validates :projects_limit, presence: true, numericality: { greater_than_or_equal_to: 0 } @@ -117,7 +123,7 @@ class User < ActiveRecord::Base before_validation :set_public_email, if: ->(user) { user.public_email_changed? } after_update :update_emails_with_primary_email, if: ->(user) { user.email_changed? } - before_save :ensure_authentication_token + before_save :ensure_authentication_token, :ensure_incoming_email_token before_save :ensure_external_user_rights after_save :ensure_namespace_correct after_initialize :set_projects_limit @@ -170,7 +176,8 @@ class User < ActiveRecord::Base scope :external, -> { where(external: true) } scope :active, -> { with_state(:active) } scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all } - scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members)') } + scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') } + scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) } def self.with_two_factor joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id"). @@ -222,19 +229,19 @@ class User < ActiveRecord::Base def filter(filter_name) case filter_name when 'admins' - self.admins + admins when 'blocked' - self.blocked + blocked when 'two_factor_disabled' - self.without_two_factor + without_two_factor when 'two_factor_enabled' - self.with_two_factor + with_two_factor when 'wop' - self.without_projects + without_projects when 'external' - self.external + external else - self.active + active end end @@ -256,6 +263,24 @@ class User < ActiveRecord::Base ) end + # searches user by given pattern + # it compares name, email, username fields and user's secondary emails with given pattern + # This method uses ILIKE on PostgreSQL and LIKE on MySQL. + + def search_with_secondary_emails(query) + table = arel_table + email_table = Email.arel_table + pattern = "%#{query}%" + matched_by_emails_user_ids = email_table.project(email_table[:user_id]).where(email_table[:email].matches(pattern)) + + where( + table[:name].matches(pattern). + or(table[:email].matches(pattern)). + or(table[:username].matches(pattern)). + or(table[:id].in(matched_by_emails_user_ids)) + ) + end + def by_login(login) return nil unless login @@ -266,8 +291,12 @@ class User < ActiveRecord::Base end end + def find_by_username(username) + iwhere(username: username).take + end + def find_by_username!(username) - find_by!('lower(username) = ?', username.downcase) + iwhere(username: username).take! end def find_by_personal_access_token(token_string) @@ -279,6 +308,11 @@ class User < ActiveRecord::Base find_by('users.username = ? OR users.id = ?', name_or_id.to_s, name_or_id.to_i) end + # Returns a user for the given SSH key. + def find_by_ssh_key_id(key_id) + find_by(id: Key.unscoped.select(:user_id).where(id: key_id)) + end + def build_user(attrs = {}) User.new(attrs) end @@ -304,12 +338,12 @@ class User < ActiveRecord::Base username end - def to_reference(_from_project = nil) + def to_reference(_from_project = nil, _target_project = nil) "#{self.class.reference_prefix}#{username}" end def generate_password - if self.force_random_password + if force_random_password self.password = self.password_confirmation = Devise.friendly_token.first(Devise.password_length.min) end end @@ -350,56 +384,55 @@ class User < ActiveRecord::Base end def two_factor_otp_enabled? - self.otp_required_for_login? + otp_required_for_login? end def two_factor_u2f_enabled? - self.u2f_registrations.exists? + u2f_registrations.exists? end def namespace_uniq # Return early if username already failed the first uniqueness validation - return if self.errors.key?(:username) && - self.errors[:username].include?('has already been taken') + return if errors.key?(:username) && + errors[:username].include?('has already been taken') - namespace_name = self.username - existing_namespace = Namespace.by_path(namespace_name) - if existing_namespace && existing_namespace != self.namespace - self.errors.add(:username, 'has already been taken') + existing_namespace = Namespace.by_path(username) + if existing_namespace && existing_namespace != namespace + errors.add(:username, 'has already been taken') end end def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" + unless avatar.image? + errors.add :avatar, "only images allowed" end end def unique_email - if !self.emails.exists?(email: self.email) && Email.exists?(email: self.email) - self.errors.add(:email, 'has already been taken') + if !emails.exists?(email: email) && Email.exists?(email: email) + errors.add(:email, 'has already been taken') end end def owns_notification_email - return if self.temp_oauth_email? + return if temp_oauth_email? - self.errors.add(:notification_email, "is not an email you own") unless self.all_emails.include?(self.notification_email) + errors.add(:notification_email, "is not an email you own") unless all_emails.include?(notification_email) end def owns_public_email - return if self.public_email.blank? + return if public_email.blank? - self.errors.add(:public_email, "is not an email you own") unless self.all_emails.include?(self.public_email) + errors.add(:public_email, "is not an email you own") unless all_emails.include?(public_email) end def update_emails_with_primary_email - primary_email_record = self.emails.find_by(email: self.email) + primary_email_record = emails.find_by(email: email) if primary_email_record primary_email_record.destroy - self.emails.create(email: self.email_was) + emails.create(email: email_was) - self.update_secondary_emails! + update_secondary_emails! end end @@ -411,16 +444,60 @@ class User < ActiveRecord::Base Group.where("namespaces.id IN (#{union.to_sql})") end - # Returns projects user is authorized to access. - # - # If you change the logic of this method, please also update `Project#authorized_for_user` + def refresh_authorized_projects + loop do + begin + Gitlab::Database.serialized_transaction do + project_authorizations.delete_all + + # project_authorizations_union can return multiple records for the same project/user with + # different access_level so we take row with the maximum access_level + project_authorizations.connection.execute <<-SQL + INSERT INTO project_authorizations (user_id, project_id, access_level) + SELECT user_id, project_id, MAX(access_level) AS access_level + FROM (#{project_authorizations_union.to_sql}) sub + GROUP BY user_id, project_id + SQL + + update_column(:authorized_projects_populated, true) unless authorized_projects_populated + end + + break + # In the event of a concurrent modification Rails raises StatementInvalid. + # In this case we want to keep retrying until the transaction succeeds + rescue ActiveRecord::StatementInvalid + end + end + end + def authorized_projects(min_access_level = nil) - Project.where("projects.id IN (#{projects_union(min_access_level).to_sql})") + refresh_authorized_projects unless authorized_projects_populated + + # We're overriding an association, so explicitly call super with no arguments or it would be passed as `force_reload` to the association + projects = super() + projects = projects.where('project_authorizations.access_level >= ?', min_access_level) if min_access_level + + projects + end + + def authorized_project?(project, min_access_level = nil) + authorized_projects(min_access_level).exists?({ id: project.id }) + end + + # Returns the projects this user has reporter (or greater) access to, limited + # to at most the given projects. + # + # This method is useful when you have a list of projects and want to + # efficiently check to which of these projects the user has at least reporter + # access. + def projects_with_reporter_access_limited_to(projects) + authorized_projects(Gitlab::Access::REPORTER).where(id: projects) end def viewable_starred_projects - starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (#{projects_union.to_sql})", - [Project::PUBLIC, Project::INTERNAL]) + starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (?)", + [Project::PUBLIC, Project::INTERNAL], + authorized_projects.select(:project_id)) end def owned_projects @@ -543,7 +620,7 @@ class User < ActiveRecord::Base end def project_deploy_keys - DeployKey.unscoped.in_projects(self.authorized_projects.pluck(:id)).distinct(:id) + DeployKey.unscoped.in_projects(authorized_projects.pluck(:id)).distinct(:id) end def accessible_deploy_keys @@ -559,33 +636,38 @@ class User < ActiveRecord::Base end def sanitize_attrs - %w(name username skype linkedin twitter).each do |attr| - value = self.send(attr) - self.send("#{attr}=", Sanitize.clean(value)) if value.present? + %w[name username skype linkedin twitter].each do |attr| + value = public_send(attr) + public_send("#{attr}=", Sanitize.clean(value)) if value.present? end end def set_notification_email - if self.notification_email.blank? || !self.all_emails.include?(self.notification_email) - self.notification_email = self.email + if notification_email.blank? || !all_emails.include?(notification_email) + self.notification_email = email end end def set_public_email - if self.public_email.blank? || !self.all_emails.include?(self.public_email) + if public_email.blank? || !all_emails.include?(public_email) self.public_email = '' end end def update_secondary_emails! - self.set_notification_email - self.set_public_email - self.save if self.notification_email_changed? || self.public_email_changed? + set_notification_email + set_public_email + save if notification_email_changed? || public_email_changed? end def set_projects_limit + # `User.select(:id)` raises + # `ActiveModel::MissingAttributeError: missing attribute: projects_limit` + # without this safeguard! + return unless has_attribute?(:projects_limit) + connection_default_value_defined = new_record? && !projects_limit_changed? - return unless self.projects_limit.nil? || connection_default_value_defined + return unless projects_limit.nil? || connection_default_value_defined self.projects_limit = current_application_settings.default_projects_limit end @@ -615,7 +697,7 @@ class User < ActiveRecord::Base def with_defaults User.defaults.each do |k, v| - self.send("#{k}=", v) + public_send("#{k}=", v) end self @@ -635,7 +717,7 @@ class User < ActiveRecord::Base # Thus it will automatically generate a new fragment # when the event is updated because the key changes. def reset_events_cache - Event.where(author_id: self.id). + Event.where(author_id: id). order('id DESC').limit(1000). update_all(updated_at: Time.now) end @@ -668,8 +750,8 @@ class User < ActiveRecord::Base def all_emails all_emails = [] - all_emails << self.email unless self.temp_oauth_email? - all_emails.concat(self.emails.map(&:email)) + all_emails << email unless temp_oauth_email? + all_emails.concat(emails.map(&:email)) all_emails end @@ -683,21 +765,21 @@ class User < ActiveRecord::Base def ensure_namespace_correct # Ensure user has namespace - self.create_namespace!(path: self.username, name: self.username) unless self.namespace + create_namespace!(path: username, name: username) unless namespace - if self.username_changed? - self.namespace.update_attributes(path: self.username, name: self.username) + if username_changed? + namespace.update_attributes(path: username, name: username) end end def post_create_hook - log_info("User \"#{self.name}\" (#{self.email}) was created") - notification_service.new_user(self, @reset_token) if self.created_by_id + log_info("User \"#{name}\" (#{email}) was created") + notification_service.new_user(self, @reset_token) if created_by_id system_hook_service.execute_hooks_for(self, :create) end def post_destroy_hook - log_info("User \"#{self.name}\" (#{self.email}) was removed") + log_info("User \"#{name}\" (#{email}) was removed") system_hook_service.execute_hooks_for(self, :destroy) end @@ -741,7 +823,7 @@ class User < ActiveRecord::Base end def oauth_authorized_tokens - Doorkeeper::AccessToken.where(resource_owner_id: self.id, revoked_at: nil) + Doorkeeper::AccessToken.where(resource_owner_id: id, revoked_at: nil) end # Returns the projects a user contributed to in the last year. @@ -827,18 +909,32 @@ class User < ActiveRecord::Base todos_pending_count(force: true) end - private + # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth + # flow means we don't call that automatically (and can't conveniently do so). + # + # See: + # <https://github.com/plataformatec/devise/blob/v4.0.0/lib/devise/models/lockable.rb#L92> + # + def increment_failed_attempts! + self.failed_attempts ||= 0 + self.failed_attempts += 1 + if attempts_exceeded? + lock_access! unless access_locked? + else + save(validate: false) + end + end - def projects_union(min_access_level = nil) - relations = [personal_projects.select(:id), - groups_projects.select(:id), - projects.select(:id), - groups.joins(:shared_projects).select(:project_id)] + private - if min_access_level - scope = { access_level: Gitlab::Access.all_values.select { |access| access >= min_access_level } } - relations = [relations.shift] + relations.map { |relation| relation.where(members: scope) } - end + # Returns a union query of projects that the user is authorized to access + def project_authorizations_union + relations = [ + personal_projects.select("#{id} AS user_id, projects.id AS project_id, #{Gitlab::Access::MASTER} AS access_level"), + groups_projects.select_for_project_authorization, + projects.select_for_project_authorization, + groups.joins(:shared_projects).select_for_project_authorization + ] Gitlab::SQL::Union.new(relations) end @@ -858,7 +954,7 @@ class User < ActiveRecord::Base end def ensure_external_user_rights - return unless self.external? + return unless external? self.can_create_group = false self.projects_limit = 0 @@ -870,7 +966,7 @@ class User < ActiveRecord::Base if current_application_settings.domain_blacklist_enabled? blocked_domains = current_application_settings.domain_blacklist - if domain_matches?(blocked_domains, self.email) + if domain_matches?(blocked_domains, email) error = 'is not from an allowed domain.' valid = false end @@ -878,15 +974,15 @@ class User < ActiveRecord::Base allowed_domains = current_application_settings.domain_whitelist unless allowed_domains.blank? - if domain_matches?(allowed_domains, self.email) + if domain_matches?(allowed_domains, email) valid = true else - error = "is not whitelisted. Email domains valid for registration are: #{allowed_domains.join(', ')}" + error = "domain is not authorized for sign-up" valid = false end end - self.errors.add(:email, error) unless valid + errors.add(:email, error) unless valid valid end @@ -899,4 +995,13 @@ class User < ActiveRecord::Base signup_domain =~ regexp end end + + def generate_token(token_field) + if token_field == :incoming_email_token + # Needs to be all lowercase and alphanumeric because it's gonna be used in an email address. + SecureRandom.hex.to_i(16).to_s(36) + else + super + end + end end |