diff options
Diffstat (limited to 'app/models')
115 files changed, 3274 insertions, 1739 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c99aa7772bb..8fab77cda0a 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -18,6 +18,8 @@ class ApplicationSetting < ActiveRecord::Base serialize :disabled_oauth_sign_in_sources, Array serialize :domain_whitelist, Array serialize :domain_blacklist, Array + serialize :repository_storages + serialize :sidekiq_throttling_queues, Array cache_markdown_field :sign_in_text cache_markdown_field :help_page_text @@ -66,6 +68,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, if: :koding_enabled + validates :plantuml_url, + presence: true, + if: :plantuml_enabled + validates :max_attachment_size, presence: true, numericality: { only_integer: true, greater_than: 0 } @@ -74,9 +80,8 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: 0 } - validates :repository_storage, - presence: true, - inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + validates :repository_storages, presence: true + validate :check_repository_storages validates :enabled_git_access_protocol, inclusion: { in: %w(ssh http), allow_blank: true, allow_nil: true } @@ -85,6 +90,27 @@ class ApplicationSetting < ActiveRecord::Base presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' }, if: :domain_blacklist_enabled? + validates :sidekiq_throttling_factor, + numericality: { greater_than: 0, less_than: 1 }, + presence: { message: 'Throttling factor cannot be empty if Sidekiq Throttling is enabled.' }, + if: :sidekiq_throttling_enabled? + + validates :sidekiq_throttling_queues, + presence: { message: 'Queues to throttle cannot be empty if Sidekiq Throttling is enabled.' }, + if: :sidekiq_throttling_enabled? + + validates :housekeeping_incremental_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: 0 } + + validates :housekeeping_full_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_incremental_repack_period } + + validates :housekeeping_gc_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_full_repack_period } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -162,12 +188,20 @@ class ApplicationSetting < ActiveRecord::Base akismet_enabled: false, koding_enabled: false, koding_url: nil, + plantuml_enabled: false, + plantuml_url: nil, repository_checks_enabled: true, disabled_oauth_sign_in_sources: [], send_user_confirmation_email: false, container_registry_token_expire_delay: 5, - repository_storage: 'default', + repository_storages: ['default'], user_default_external: false, + sidekiq_throttling_enabled: false, + housekeeping_enabled: true, + housekeeping_bitmaps_enabled: true, + housekeeping_incremental_repack_period: 10, + housekeeping_full_repack_period: 50, + housekeeping_gc_period: 200, ) end @@ -175,6 +209,10 @@ class ApplicationSetting < ActiveRecord::Base ActiveRecord::Base.connection.column_exists?(:application_settings, :home_page_url) end + def sidekiq_throttling_column_exists? + ActiveRecord::Base.connection.column_exists?(:application_settings, :sidekiq_throttling_enabled) + end + def domain_whitelist_raw self.domain_whitelist.join("\n") unless self.domain_whitelist.nil? end @@ -201,6 +239,25 @@ class ApplicationSetting < ActiveRecord::Base self.domain_blacklist_raw = file.read end + def repository_storages + Array(read_attribute(:repository_storages)) + end + + # repository_storage is still required in the API. Remove in 9.0 + def repository_storage + repository_storages.first + end + + def repository_storage=(value) + self.repository_storages = [value] + end + + # Choose one of the available repository storage options. Currently all have + # equal weighting. + def pick_repository_storage + repository_storages.sample + end + def runners_registration_token ensure_runners_registration_token! end @@ -208,4 +265,18 @@ class ApplicationSetting < ActiveRecord::Base def health_check_access_token ensure_health_check_access_token! end + + def sidekiq_throttling_enabled? + return false unless sidekiq_throttling_column_exists? + + sidekiq_throttling_enabled + end + + private + + def check_repository_storages + invalid = repository_storages - Gitlab.config.repositories.storages.keys + errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless + invalid.empty? + end end diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb new file mode 100644 index 00000000000..f321db75eeb --- /dev/null +++ b/app/models/chat_name.rb @@ -0,0 +1,12 @@ +class ChatName < ActiveRecord::Base + belongs_to :service + belongs_to :user + + validates :user, presence: true + validates :service, presence: true + validates :team_id, presence: true + validates :chat_id, presence: true + + validates :user_id, uniqueness: { scope: [:service_id] } + validates :chat_id, uniqueness: { scope: [:service_id, :team_id] } +end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index bf5f92f8462..5fe8ddf69d7 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -2,13 +2,24 @@ module Ci class Build < CommitStatus include TokenAuthenticatable include AfterCommitQueue + include Presentable belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' + has_many :deployments, as: :deployable + + # The "environment" field for builds is a String, and is the unexpanded name + def persisted_environment + @persisted_environment ||= Environment.find_by( + name: expanded_environment_name, + project_id: gl_project_id + ) + end + serialize :options - serialize :yaml_variables + serialize :yaml_variables, Gitlab::Serialize::Ci::Variables validates :coverage, numericality: true, allow_blank: true validates_presence_of :ref @@ -33,6 +44,8 @@ module Ci before_destroy { project } after_create :execute_hooks + after_save :update_project_statistics, if: :artifacts_size_changed? + after_destroy :update_project_statistics class << self def first_pending @@ -68,13 +81,23 @@ module Ci environment: build.environment, status_event: 'enqueue' ) - MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build) + + MergeRequests::AddTodoWhenBuildFailsService + .new(build.project, nil) + .close(new_build) + build.pipeline.mark_as_processable_after_stage(build.stage_idx) new_build end end state_machine :status do + after_transition any => [:pending] do |build| + build.run_after_commit do + BuildQueueWorker.perform_async(id) + end + end + after_transition pending: :running do |build| build.run_after_commit do BuildHooksWorker.perform_async(id) @@ -94,6 +117,12 @@ module Ci end end + def detailed_status(current_user) + Gitlab::Ci::Status::Build::Factory + .new(self, current_user) + .fabricate! + end + def manual? self.when == 'manual' end @@ -117,14 +146,47 @@ module Ci end end + def cancelable? + active? + end + def retryable? - project.builds_enabled? && commands.present? && complete? + project.builds_enabled? && commands.present? && + (success? || failed? || canceled?) end def retried? !self.pipeline.statuses.latest.include?(self) end + def expanded_environment_name + ExpandVariables.expand(environment, simple_variables) if environment + end + + def has_environment? + environment.present? + end + + def starts_environment? + has_environment? && self.environment_action == 'start' + end + + def stops_environment? + has_environment? && self.environment_action == 'stop' + end + + def environment_action + self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options + end + + def outdated_deployment? + success? && !last_deployment.try(:last?) + end + + def last_deployment + deployments.last + end + def depends_on_builds # Get builds of the same type latest_builds = self.pipeline.builds.latest @@ -150,12 +212,25 @@ module Ci project.build_timeout end - def variables + # A slugified version of the build ref, suitable for inclusion in URLs and + # domain names. Rules: + # + # * Lowercased + # * Anything not matching [a-z0-9-] is replaced with a - + # * Maximum length is 63 bytes + def ref_slug + slugified = ref.to_s.downcase + slugified.gsub(/[^a-z0-9]/, '-')[0..62] + end + + # Variables whose value does not depend on other variables + def simple_variables variables = predefined_variables variables += project.predefined_variables variables += pipeline.predefined_variables variables += runner.predefined_variables if runner variables += project.container_registry_variables + variables += project.deployment_variables if has_environment? variables += yaml_variables variables += user_variables variables += project.secret_variables @@ -163,13 +238,20 @@ module Ci variables end + # All variables, including those dependent on other variables + def variables + variables = simple_variables + variables += persisted_environment.predefined_variables if persisted_environment.present? + variables + end + def merge_request merge_requests = MergeRequest.includes(:merge_request_diff) .where(source_branch: ref, source_project_id: pipeline.gl_project_id) .reorder(iid: :asc) merge_requests.find do |merge_request| - merge_request.commits.any? { |ci| ci.id == pipeline.sha } + merge_request.commits_sha.include?(pipeline.sha) end end @@ -271,6 +353,7 @@ module Ci def append_trace(trace_part, offset) recreate_trace_dir + touch if needs_touch? trace_part = hide_secrets(trace_part) @@ -280,6 +363,10 @@ module Ci end end + def needs_touch? + Time.now - updated_at > 15.minutes.to_i + end + def trace_file_path if has_old_trace_file? old_path_to_trace @@ -427,6 +514,10 @@ module Ci end end + def has_expiring_artifacts? + artifacts_expire_at.present? + end + def keep_artifacts! self.update(artifacts_expire_at: nil) end @@ -448,6 +539,10 @@ module Ci ] end + def credentials + Gitlab::Ci::Build::Credentials::Factory.new(self).create! + end + private def update_artifacts_size @@ -475,6 +570,7 @@ module Ci { key: 'CI_BUILD_REF', value: sha, public: true }, { key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true }, { key: 'CI_BUILD_REF_NAME', value: ref, public: true }, + { key: 'CI_BUILD_REF_SLUG', value: ref_slug, public: true }, { key: 'CI_BUILD_NAME', value: name, public: true }, { key: 'CI_BUILD_STAGE', value: stage, public: true }, { key: 'CI_SERVER_NAME', value: 'GitLab', public: true }, @@ -501,5 +597,9 @@ module Ci Ci::MaskSecret.mask!(trace, token) trace end + + def update_project_statistics + ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size]) + end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index d3432632899..fab8497ec7d 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -21,8 +21,6 @@ module Ci after_create :keep_around_commits, unless: :importing? - delegate :stages, to: :statuses - state_machine :status, initial: :created do event :enqueue do transition created: :pending @@ -81,34 +79,86 @@ module Ci PipelineHooksWorker.perform_async(id) end end + + after_transition any => [:success, :failed] do |pipeline| + pipeline.run_after_commit do + PipelineNotificationWorker.perform_async(pipeline.id) + end + end end # ref can't be HEAD or SHA, can only be branch/tag name + scope :latest, ->(ref = nil) do + max_id = unscope(:select) + .select("max(#{quoted_table_name}.id)") + .group(:ref, :sha) + + relation = ref ? where(ref: ref) : self + relation.where(id: max_id) + end + + def self.latest_status(ref = nil) + latest(ref).status + end + def self.latest_successful_for(ref) - where(ref: ref).order(id: :desc).success.first + success.latest(ref).order(id: :desc).first end def self.truncate_sha(sha) sha[0...8] end - def self.stages - # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries - CommitStatus.where(pipeline: pluck(:id)).stages - end - def self.total_duration where.not(duration: nil).sum(:duration) end - def stages_with_latest_statuses - statuses.latest.includes(project: :namespace).order(:stage_idx).group_by(&:stage) + def stage(name) + stage = Ci::Stage.new(self, name: name) + stage unless stage.statuses_count.zero? + end + + def stages_count + statuses.select(:stage).distinct.count + end + + def stages_name + statuses.order(:stage_idx).distinct. + pluck(:stage, :stage_idx).map(&:first) + end + + def stages + # TODO, this needs refactoring, see gitlab-ce#26481. + + stages_query = statuses + .group('stage').select(:stage).order('max(stage_idx)') + + status_sql = statuses.latest.where('stage=sg.stage').status_sql + + warnings_sql = statuses.latest.select('COUNT(*) > 0') + .where('stage=sg.stage').failed_but_allowed.to_sql + + stages_with_statuses = CommitStatus.from(stages_query, :sg) + .pluck('sg.stage', status_sql, "(#{warnings_sql})") + + stages_with_statuses.map do |stage| + Ci::Stage.new(self, Hash[%i[name status warnings].zip(stage)]) + end + end + + def artifacts + builds.latest.with_artifacts_not_expired.includes(project: [:namespace]) end def project_id project.id end + # For now the only user who participates is the user who triggered + def participants(_current_user = nil) + Array(user) + end + def valid_commit_sha if self.sha == Gitlab::Git::BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") @@ -146,27 +196,35 @@ module Ci end def manual_actions - builds.latest.manual_actions + builds.latest.manual_actions.includes(project: [:namespace]) + end + + def stuck? + builds.pending.any?(&:stuck?) end def retryable? - builds.latest.any? do |build| - (build.failed? || build.canceled?) && build.retryable? - end + builds.latest.failed_or_canceled.any?(&:retryable?) end def cancelable? - builds.running_or_pending.any? + statuses.cancelable.any? end def cancel_running - builds.running_or_pending.each(&:cancel) + Gitlab::OptimisticLocking.retry_lock( + statuses.cancelable) do |cancelable| + cancelable.each(&:cancel) + end end def retry_failed(user) - builds.latest.failed.select(&:retryable?).each do |build| - Ci::Build.retry(build, user) - end + Gitlab::OptimisticLocking.retry_lock( + builds.latest.failed_or_canceled) do |failed_or_canceled| + failed_or_canceled.select(&:retryable?).each do |build| + Ci::Build.retry(build, user) + end + end end def mark_as_processable_after_stage(stage_idx) @@ -234,6 +292,10 @@ module Ci end end + def has_yaml_errors? + yaml_errors.present? + end + def environments builds.where.not(environment: nil).success.pluck(:environment).uniq end @@ -302,7 +364,13 @@ module Ci def merge_requests @merge_requests ||= project.merge_requests .where(source_branch: self.ref) - .select { |merge_request| merge_request.pipeline.try(:id) == self.id } + .select { |merge_request| merge_request.head_pipeline.try(:id) == self.id } + end + + def detailed_status(current_user) + Gitlab::Ci::Status::Pipeline::Factory + .new(self, current_user) + .fabricate! end private diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 123930273e0..ed1843ba005 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -2,6 +2,7 @@ module Ci class Runner < ActiveRecord::Base extend Ci::Model + RUNNER_QUEUE_EXPIRY_TIME = 60.minutes LAST_CONTACT_TIME = 1.hour.ago AVAILABLE_SCOPES = %w[specific shared active paused online] FORM_EDITABLE = %i[description tag_list active run_untagged locked] @@ -21,6 +22,8 @@ module Ci scope :online, ->() { where('contacted_at > ?', LAST_CONTACT_TIME) } scope :ordered, ->() { order(id: :desc) } + after_save :tick_runner_queue, if: :form_editable_changed? + scope :owned_or_shared, ->(project_id) do joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id') .where("ci_runner_projects.gl_project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id) @@ -122,8 +125,38 @@ module Ci ] end + def tick_runner_queue + SecureRandom.hex.tap do |new_update| + Gitlab::Redis.with do |redis| + redis.set(runner_queue_key, new_update, ex: RUNNER_QUEUE_EXPIRY_TIME) + end + end + end + + def ensure_runner_queue_value + Gitlab::Redis.with do |redis| + value = SecureRandom.hex + redis.set(runner_queue_key, value, ex: RUNNER_QUEUE_EXPIRY_TIME, nx: true) + redis.get(runner_queue_key) + end + end + + def is_runner_queue_value_latest?(value) + ensure_runner_queue_value == value if value.present? + end + private + def runner_queue_key + "runner:build_queue:#{self.token}" + end + + def form_editable_changed? + FORM_EDITABLE.any? do |editable| + public_send("#{editable}_changed?") + end + end + def tag_constraints unless has_tags? || run_untagged? errors.add(:tags_list, diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb new file mode 100644 index 00000000000..ca74c91b062 --- /dev/null +++ b/app/models/ci/stage.rb @@ -0,0 +1,56 @@ +module Ci + # Currently this is artificial object, constructed dynamically + # We should migrate this object to actual database record in the future + class Stage + include StaticModel + + attr_reader :pipeline, :name + + delegate :project, to: :pipeline + + def initialize(pipeline, name:, status: nil, warnings: nil) + @pipeline = pipeline + @name = name + @status = status + @warnings = warnings + end + + def to_param + name + end + + def statuses_count + @statuses_count ||= statuses.count + end + + def status + @status ||= statuses.latest.status + end + + def detailed_status(current_user) + Gitlab::Ci::Status::Stage::Factory + .new(self, current_user) + .fabricate! + end + + def statuses + @statuses ||= pipeline.statuses.where(stage: name) + end + + def builds + @builds ||= pipeline.builds.where(stage: name) + end + + def success? + status.to_s == 'success' + end + + def has_warnings? + if @warnings.nil? + statuses.latest.failed_but_allowed.any? + else + @warnings + end + end + end +end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 94d9e2b3208..2c8698d8b5d 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -4,10 +4,10 @@ module Ci belongs_to :project, foreign_key: :gl_project_id - validates_uniqueness_of :key, scope: :gl_project_id validates :key, presence: true, - length: { within: 0..255 }, + uniqueness: { scope: :gl_project_id }, + length: { maximum: 255 }, format: { with: /\A[a-zA-Z0-9_]+\z/, message: "can contain only letters, digits and '_'." } diff --git a/app/models/commit.rb b/app/models/commit.rb index e64fd1e0c1b..5d942cb0422 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -48,6 +48,10 @@ class Commit max_lines: DIFF_HARD_LIMIT_LINES, } end + + def from_hash(hash, project) + new(Gitlab::Git::Commit.new(hash), project) + end end attr_accessor :raw @@ -87,20 +91,12 @@ class Commit @link_reference_pattern ||= super("commit", /(?<commit>\h{7,40})/) end - def to_reference(from_project = nil) - if cross_project_reference?(from_project) - project.to_reference + self.class.reference_prefix + self.id - else - self.id - end + def to_reference(from_project = nil, full: false) + commit_reference(from_project, id, full: full) end def reference_link_text(from_project = nil) - if cross_project_reference?(from_project) - project.to_reference + self.class.reference_prefix + self.short_id - else - self.short_id - end + commit_reference(from_project, short_id) end def diff_line_count @@ -226,12 +222,15 @@ class Commit end def pipelines - @pipeline ||= project.pipelines.where(sha: sha) + project.pipelines.where(sha: sha) end - def status - return @status if defined?(@status) - @status ||= pipelines.status + def status(ref = nil) + @statuses ||= {} + + return @statuses[ref] if @statuses.key?(ref) + + @statuses[ref] = pipelines.latest_status(ref) end def revert_branch_name @@ -242,44 +241,47 @@ class Commit project.repository.next_branch("cherry-pick-#{short_id}", mild: true) end - def revert_description - if merged_merge_request - "This reverts merge request #{merged_merge_request.to_reference}" + def revert_description(user) + if merged_merge_request?(user) + "This reverts merge request #{merged_merge_request(user).to_reference}" else "This reverts commit #{sha}" end end - def revert_message - %Q{Revert "#{title.strip}"\n\n#{revert_description}} + def revert_message(user) + %Q{Revert "#{title.strip}"\n\n#{revert_description(user)}} end - def reverts_commit?(commit) - description? && description.include?(commit.revert_description) + def reverts_commit?(commit, user) + description? && description.include?(commit.revert_description(user)) end def merge_commit? parents.size > 1 end - def merged_merge_request - return @merged_merge_request if defined?(@merged_merge_request) + def merged_merge_request(current_user) + # Memoize with per-user access check + @merged_merge_request_hash ||= Hash.new do |hash, user| + hash[user] = merged_merge_request_no_cache(user) + end - @merged_merge_request = project.merge_requests.find_by(merge_commit_sha: id) if merge_commit? + @merged_merge_request_hash[current_user] end - def has_been_reverted?(current_user = nil, noteable = self) + def has_been_reverted?(current_user, noteable = self) ext = all_references(current_user) noteable.notes_with_associations.system.each do |note| note.all_references(current_user, extractor: ext) end - ext.commits.any? { |commit_ref| commit_ref.reverts_commit?(self) } + ext.commits.any? { |commit_ref| commit_ref.reverts_commit?(self, current_user) } end - def change_type_title - merged_merge_request ? 'merge request' : 'commit' + def change_type_title(user) + merged_merge_request?(user) ? 'merge request' : 'commit' end # Get the URI type of the given path @@ -316,8 +318,32 @@ class Commit Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options) end + def persisted? + true + end + + def touch + # no-op but needs to be defined since #persisted? is defined + end + + WIP_REGEX = /\A\s*(((?i)(\[WIP\]|WIP:|WIP)\s|WIP$))|(fixup!|squash!)\s/.freeze + + def work_in_progress? + !!(title =~ WIP_REGEX) + end + private + def commit_reference(from_project, referable_commit_id, full: false) + reference = project.to_reference(from_project, full: full) + + if reference.present? + "#{reference}#{self.class.reference_prefix}#{referable_commit_id}" + else + referable_commit_id + end + end + def find_author_by_any_email User.find_by_any_email(author_email.downcase) end @@ -337,4 +363,12 @@ class Commit changes end + + def merged_merge_request?(user) + !!merged_merge_request(user) + end + + def merged_merge_request_no_cache(user) + MergeRequestsFinder.new(user, project_id: project.id).find_by(merge_commit_sha: id) if merge_commit? + end end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index ac2477fd973..84e2e8a5dd5 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -89,22 +89,25 @@ class CommitRange alias_method :id, :to_s - def to_reference(from_project = nil) - if cross_project_reference?(from_project) - project.to_reference + self.class.reference_prefix + self.id + def to_reference(from_project = nil, full: false) + project_reference = project.to_reference(from_project, full: full) + + if project_reference.present? + project_reference + self.class.reference_prefix + self.id else self.id end end def reference_link_text(from_project = nil) - reference = ref_from + notation + ref_to + project_reference = project.to_reference(from_project) + reference = ref_from + notation + ref_to - if cross_project_reference?(from_project) - reference = project.to_reference + self.class.reference_prefix + reference + if project_reference.present? + project_reference + self.class.reference_prefix + reference + else + reference end - - reference end # Return a Hash of parameters for passing to a URL helper diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d159fc6c5c7..9547c57b2ae 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -31,18 +31,13 @@ class CommitStatus < ActiveRecord::Base end scope :exclude_ignored, -> do - quoted_when = connection.quote_column_name('when') # We want to ignore failed_but_allowed jobs where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled]). - # We want to ignore skipped manual jobs - where("#{quoted_when} <> ? OR status <> ?", 'manual', 'skipped'). - # We want to ignore skipped on_failure - where("#{quoted_when} <> ? OR status <> ?", 'on_failure', 'skipped') + false, all_state_names - [:failed, :canceled]) end - scope :latest_ci_stages, -> { latest.ordered.includes(project: :namespace) } - scope :retried_ci_stages, -> { retried.ordered.includes(project: :namespace) } + scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } + scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } state_machine :status do event :enqueue do @@ -117,33 +112,35 @@ class CommitStatus < ActiveRecord::Base name.gsub(/\d+[\s:\/\\]+\d+\s*/, '').strip end - def self.stages - # We group by stage name, but order stages by theirs' index - unscoped.from(all, :sg).group('stage').order('max(stage_idx)', 'stage').pluck('sg.stage') - end - - def self.stages_status - # We execute subquery for each stage to calculate a stage status - statuses = unscoped.from(all, :sg).group('stage').pluck('sg.stage', all.where('stage=sg.stage').status_sql) - statuses.inject({}) do |h, k| - h[k.first] = k.last - h - end - end - def failed_but_allowed? allow_failure? && (failed? || canceled?) end + def duration + calculate_duration + end + def playable? false end - def duration - calculate_duration + def stuck? + false end - def stuck? + def has_trace? false end + + def detailed_status(current_user) + Gitlab::Ci::Status::Factory + .new(self, current_user) + .fabricate! + end + + def sortable_name + name.split(/(\d+)/).map do |v| + v =~ /\d+/ ? v.to_i : v + end + end end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index ef3e73a4072..431c0354969 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -1,10 +1,11 @@ module HasStatus extend ActiveSupport::Concern + DEFAULT_STATUS = 'created' AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped] STARTED_STATUSES = %w[running success failed skipped] ACTIVE_STATUSES = %w[pending running] - COMPLETED_STATUSES = %w[success failed canceled] + COMPLETED_STATUSES = %w[success failed canceled skipped] ORDERED_STATUSES = %w[failed pending running canceled success skipped] class_methods do @@ -23,9 +24,10 @@ module HasStatus canceled = scope.canceled.select('count(*)').to_sql "(CASE + WHEN (#{builds})=(#{skipped}) THEN 'skipped' WHEN (#{builds})=(#{success}) THEN 'success' WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' WHEN (#{running})+(#{pending})+(#{created})>0 THEN 'running' @@ -73,6 +75,11 @@ module HasStatus scope :skipped, -> { where(status: 'skipped') } scope :running_or_pending, -> { where(status: [:running, :pending]) } scope :finished, -> { where(status: [:success, :failed, :canceled]) } + scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } + + scope :cancelable, -> do + where(status: [:running, :pending, :created]) + end end def started? diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 613444e0d70..3517969eabc 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -13,6 +13,7 @@ module Issuable include StripAttribute include Awardable include Taskable + include TimeTrackable included do cache_markdown_field :title, pipeline: :single_line @@ -41,7 +42,7 @@ module Issuable has_one :metrics validates :author, presence: true - validates :title, presence: true, length: { within: 0..255 } + validates :title, presence: true, length: { maximum: 255 } scope :authored, ->(user) { where(author_id: user) } scope :assigned_to, ->(u) { where(assignee_id: u.id)} @@ -92,8 +93,9 @@ module Issuable after_save :record_metrics def update_assignee_cache_counts - # make sure we flush the cache for both the old *and* new assignee - User.find(assignee_id_was).update_cache_counts if assignee_id_was + # make sure we flush the cache for both the old *and* new assignees(if they exist) + previous_assignee = User.find_by_id(assignee_id_was) if assignee_id_was + previous_assignee.update_cache_counts if previous_assignee assignee.update_cache_counts if assignee end @@ -183,6 +185,10 @@ module Issuable grouping_columns end + + def to_ability_name + model_name.singular + end end def today? @@ -211,7 +217,7 @@ module Issuable end end - def subscribed_without_subscriptions?(user) + def subscribed_without_subscriptions?(user, project) participants(user).include?(user) end @@ -244,7 +250,18 @@ module Issuable # issuable.class # => MergeRequest # issuable.to_ability_name # => "merge_request" def to_ability_name - self.class.to_s.underscore + self.class.to_ability_name + end + + # Convert this Issuable class name to a format usable by notifications. + # + # Examples: + # + # issuable.class # => MergeRequest + # issuable.human_class_name # => "merge request" + + def human_class_name + @human_class_name ||= self.class.name.titleize.downcase end # Returns a Hash of attributes to be used for Twitter card metadata @@ -286,6 +303,11 @@ module Issuable false end + def assignee_or_author?(user) + # We're comparing IDs here so we don't need to load any associations. + author_id == user.id || assignee_id == user.id + end + def record_metrics metrics = self.metrics || create_metrics metrics.record! diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index eb2ff0428f6..8ab0401d288 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -1,6 +1,6 @@ # == Mentionable concern # -# Contains functionality related to objects that can mention Users, Issues, MergeRequests, or Commits by +# Contains functionality related to objects that can mention Users, Issues, MergeRequests, Commits or Snippets by # GFM references. # # Used by Issue, Note, MergeRequest, and Commit. diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 7bcc78247ba..e9450dd0c26 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -1,17 +1,25 @@ module Milestoneish - def closed_items_count(user = nil) - issues_visible_to_user(user).closed.size + merge_requests.closed_and_merged.size + def closed_items_count(user) + memoize_per_user(user, :closed_items_count) do + (count_issues_by_state(user)['closed'] || 0) + merge_requests.closed_and_merged.size + end end - def total_items_count(user = nil) - issues_visible_to_user(user).size + merge_requests.size + def total_items_count(user) + memoize_per_user(user, :total_items_count) do + total_issues_count(user) + merge_requests.size + end end - def complete?(user = nil) + def total_issues_count(user) + count_issues_by_state(user).values.sum + end + + def complete?(user) total_items_count(user) > 0 && total_items_count(user) == closed_items_count(user) end - def percent_complete(user = nil) + def percent_complete(user) ((closed_items_count(user) * 100) / total_items_count(user)).abs rescue ZeroDivisionError 0 @@ -23,7 +31,54 @@ module Milestoneish (due_date - Date.today).to_i end - def issues_visible_to_user(user = nil) - issues.visible_to_user(user) + def elapsed_days + return 0 if !start_date || start_date.future? + + (Date.today - start_date).to_i + end + + def issues_visible_to_user(user) + memoize_per_user(user, :issues_visible_to_user) do + IssuesFinder.new(user, issues_finder_params) + .execute.where(milestone_id: milestoneish_ids) + end + end + + def upcoming? + start_date && start_date.future? + end + + def expires_at + if due_date + if due_date.past? + "expired on #{due_date.to_s(:medium)}" + else + "expires on #{due_date.to_s(:medium)}" + end + end + end + + def expired? + due_date && due_date.past? + end + + private + + def count_issues_by_state(user) + memoize_per_user(user, :count_issues_by_state) do + issues_visible_to_user(user).reorder(nil).group(:state).count + end + end + + def memoize_per_user(user, method_name) + @memoized ||= {} + @memoized[method_name] ||= {} + @memoized[method_name][user.try!(:id)] ||= yield + end + + # override in a class that includes this module to get a faster query + # from IssuesFinder + def issues_finder_params + {} end end diff --git a/app/models/concerns/presentable.rb b/app/models/concerns/presentable.rb new file mode 100644 index 00000000000..7b33b837004 --- /dev/null +++ b/app/models/concerns/presentable.rb @@ -0,0 +1,7 @@ +module Presentable + def present(**attributes) + Gitlab::View::Presenter::Factory + .new(self, attributes) + .fabricate! + end +end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 6d88951c713..60734bc6660 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -32,6 +32,6 @@ module ProjectFeaturesCompatibility build_project_feature unless project_feature access_level = Gitlab::Utils.to_boolean(value) ? ProjectFeature::ENABLED : ProjectFeature::DISABLED - project_feature.update_attribute(field, access_level) + project_feature.send(:write_attribute, field, access_level) end end diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index 7fd0905ee81..9dd4d9c6f24 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -2,6 +2,9 @@ module ProtectedBranchAccess extend ActiveSupport::Concern included do + belongs_to :protected_branch + delegate :project, to: :protected_branch + scope :master, -> { where(access_level: Gitlab::Access::MASTER) } scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } end @@ -9,4 +12,10 @@ module ProtectedBranchAccess def humanize self.class.human_access_levels[self.access_level] end + + def check_access(user) + return true if user.is_admin? + + project.team.max_member_access(user.id) >= access_level + end end diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb new file mode 100644 index 00000000000..2589215ad19 --- /dev/null +++ b/app/models/concerns/reactive_caching.rb @@ -0,0 +1,118 @@ +# The ReactiveCaching concern is used to fetch some data in the background and +# store it in the Rails cache, keeping it up-to-date for as long as it is being +# requested. If the data hasn't been requested for +reactive_cache_lifetime+, +# it stop being refreshed, and then be removed. +# +# Example of use: +# +# class Foo < ActiveRecord::Base +# include ReactiveCaching +# +# self.reactive_cache_key = ->(thing) { ["foo", thing.id] } +# +# after_save :clear_reactive_cache! +# +# def calculate_reactive_cache +# # Expensive operation here. The return value of this method is cached +# end +# +# def result +# with_reactive_cache do |data| +# # ... +# end +# end +# end +# +# In this example, the first time `#result` is called, it will return `nil`. +# However, it will enqueue a background worker to call `#calculate_reactive_cache` +# and set an initial cache lifetime of ten minutes. +# +# Each time the background job completes, it stores the return value of +# `#calculate_reactive_cache`. It is also re-enqueued to run again after +# `reactive_cache_refresh_interval`, so keeping the stored value up to date. +# Calculations are never run concurrently. +# +# Calling `#result` while a value is in the cache will call the block given to +# `#with_reactive_cache`, yielding the cached value. It will also extend the +# lifetime by `reactive_cache_lifetime`. +# +# Once the lifetime has expired, no more background jobs will be enqueued and +# calling `#result` will again return `nil` - starting the process all over +# again +module ReactiveCaching + extend ActiveSupport::Concern + + included do + class_attribute :reactive_cache_lease_timeout + + class_attribute :reactive_cache_key + class_attribute :reactive_cache_lifetime + class_attribute :reactive_cache_refresh_interval + + # defaults + self.reactive_cache_lease_timeout = 2.minutes + + self.reactive_cache_refresh_interval = 1.minute + self.reactive_cache_lifetime = 10.minutes + + def calculate_reactive_cache(*args) + raise NotImplementedError + end + + def with_reactive_cache(*args, &blk) + within_reactive_cache_lifetime(*args) do + data = Rails.cache.read(full_reactive_cache_key(*args)) + yield data if data.present? + end + ensure + Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) + ReactiveCachingWorker.perform_async(self.class, id, *args) + end + + def clear_reactive_cache!(*args) + Rails.cache.delete(full_reactive_cache_key(*args)) + end + + def exclusively_update_reactive_cache!(*args) + locking_reactive_cache(*args) do + within_reactive_cache_lifetime(*args) do + enqueuing_update(*args) do + value = calculate_reactive_cache(*args) + Rails.cache.write(full_reactive_cache_key(*args), value) + end + end + end + end + + private + + def full_reactive_cache_key(*qualifiers) + prefix = self.class.reactive_cache_key + prefix = prefix.call(self) if prefix.respond_to?(:call) + + ([prefix].flatten + qualifiers).join(':') + end + + def alive_reactive_cache_key(*qualifiers) + full_reactive_cache_key(*(qualifiers + ['alive'])) + end + + def locking_reactive_cache(*args) + lease = Gitlab::ExclusiveLease.new(full_reactive_cache_key(*args), timeout: reactive_cache_lease_timeout) + uuid = lease.try_obtain + yield if uuid + ensure + Gitlab::ExclusiveLease.cancel(full_reactive_cache_key(*args), uuid) + end + + def within_reactive_cache_lifetime(*args) + yield if Rails.cache.read(alive_reactive_cache_key(*args)) + end + + def enqueuing_update(*args) + yield + ensure + ReactiveCachingWorker.perform_in(self.class.reactive_cache_refresh_interval, self.class, id, *args) + end + end +end diff --git a/app/models/concerns/reactive_service.rb b/app/models/concerns/reactive_service.rb new file mode 100644 index 00000000000..e1f868a299b --- /dev/null +++ b/app/models/concerns/reactive_service.rb @@ -0,0 +1,10 @@ +module ReactiveService + extend ActiveSupport::Concern + + included do + include ReactiveCaching + + # Default cache key: class name + project_id + self.reactive_cache_key = ->(service) { [ service.class.model_name.singular, service.project_id ] } + end +end diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index dee940a3f88..da803c7f481 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -17,7 +17,7 @@ module Referable # Issue.last.to_reference(other_project) # => "cross-project#1" # # Returns a String - def to_reference(_from_project = nil) + def to_reference(_from_project = nil, full:) '' end @@ -72,17 +72,4 @@ module Referable }x end end - - private - - # Check if a reference is being done cross-project - # - # from_project - Refering Project object - def cross_project_reference?(from_project) - if self.is_a?(Project) - self != from_project - else - from_project && self.project && self.project != from_project - end - end end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb new file mode 100644 index 00000000000..1108a64c59e --- /dev/null +++ b/app/models/concerns/routable.rb @@ -0,0 +1,71 @@ +# Store object full path in separate table for easy lookup and uniq validation +# Object must have path db field and respond to full_path and full_path_changed? methods. +module Routable + extend ActiveSupport::Concern + + included do + has_one :route, as: :source, autosave: true, dependent: :destroy + + validates_associated :route + validates :route, presence: true + + before_validation :update_route_path, if: :full_path_changed? + end + + class_methods do + # Finds a single object by full path match in routes table. + # + # Usage: + # + # Klass.find_by_full_path('gitlab-org/gitlab-ce') + # + # Returns a single object, or nil. + def find_by_full_path(path) + # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so + # any literal matches come first, for this we have to use "BINARY". + # Without this there's still no guarantee in what order MySQL will return + # rows. + binary = Gitlab::Database.mysql? ? 'BINARY' : '' + + order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + + where_full_path_in([path]).reorder(order_sql).take + end + + # Builds a relation to find multiple objects by their full paths. + # + # Usage: + # + # Klass.where_full_path_in(%w{gitlab-org/gitlab-ce gitlab-org/gitlab-ee}) + # + # Returns an ActiveRecord::Relation. + def where_full_path_in(paths) + wheres = [] + cast_lower = Gitlab::Database.postgresql? + + paths.each do |path| + path = connection.quote(path) + where = "(routes.path = #{path})" + + if cast_lower + where = "(#{where} OR (LOWER(routes.path) = LOWER(#{path})))" + end + + wheres << where + end + + if wheres.empty? + none + else + joins(:route).where(wheres.join(' OR ')) + end + end + end + + private + + def update_route_path + route || build_route(source: self) + route.path = full_path + end +end diff --git a/app/models/concerns/select_for_project_authorization.rb b/app/models/concerns/select_for_project_authorization.rb new file mode 100644 index 00000000000..50a1d7fc3e1 --- /dev/null +++ b/app/models/concerns/select_for_project_authorization.rb @@ -0,0 +1,9 @@ +module SelectForProjectAuthorization + extend ActiveSupport::Concern + + module ClassMethods + def select_for_project_authorization + select("members.user_id, projects.id AS project_id, members.access_level") + end + end +end diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 083257f1005..83daa9b1a64 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -12,39 +12,71 @@ module Subscribable has_many :subscriptions, dependent: :destroy, as: :subscribable end - def subscribed?(user) - if subscription = subscriptions.find_by_user_id(user.id) + def subscribed?(user, project = nil) + if subscription = subscriptions.find_by(user: user, project: project) subscription.subscribed else - subscribed_without_subscriptions?(user) + subscribed_without_subscriptions?(user, project) end end # Override this method to define custom logic to consider a subscribable as # subscribed without an explicit subscription record. - def subscribed_without_subscriptions?(user) + def subscribed_without_subscriptions?(user, project) false end - def subscribers - subscriptions.where(subscribed: true).map(&:user) + def subscribers(project) + subscriptions_available(project). + where(subscribed: true). + map(&:user) end - def toggle_subscription(user) - subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: !subscribed?(user)) + def toggle_subscription(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project). + update(subscribed: !subscribed?(user, project)) + end + + def subscribe(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project) + .update(subscribed: true) + end + + def unsubscribe(user, project = nil) + unsubscribe_from_other_levels(user, project) + + find_or_initialize_subscription(user, project) + .update(subscribed: false) end - def subscribe(user) + private + + def unsubscribe_from_other_levels(user, project) + other_subscriptions = subscriptions.where(user: user) + + other_subscriptions = + if project.blank? + other_subscriptions.where.not(project: nil) + else + other_subscriptions.where(project: nil) + end + + other_subscriptions.update_all(subscribed: false) + end + + def find_or_initialize_subscription(user, project) subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: true) + find_or_initialize_by(user_id: user.id, project_id: project.try(:id)) end - def unsubscribe(user) + def subscriptions_available(project) + t = Subscription.arel_table + subscriptions. - find_or_initialize_by(user_id: user.id). - update(subscribed: false) + where(t[:project_id].eq(nil).or(t[:project_id].eq(project.try(:id)))) end end diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb new file mode 100644 index 00000000000..040e3a2884e --- /dev/null +++ b/app/models/concerns/time_trackable.rb @@ -0,0 +1,72 @@ +# == TimeTrackable concern +# +# Contains functionality related to objects that support time tracking. +# +# Used by Issue and MergeRequest. +# + +module TimeTrackable + extend ActiveSupport::Concern + + included do + attr_reader :time_spent, :time_spent_user + + alias_method :time_spent?, :time_spent + + default_value_for :time_estimate, value: 0, allows_nil: false + + validates :time_estimate, numericality: { message: 'has an invalid format' }, allow_nil: false + validate :check_negative_time_spent + + has_many :timelogs, as: :trackable, dependent: :destroy + end + + def spend_time(options) + @time_spent = options[:duration] + @time_spent_user = options[:user] + @original_total_time_spent = nil + + return if @time_spent == 0 + + if @time_spent == :reset + reset_spent_time + else + add_or_subtract_spent_time + end + end + alias_method :spend_time=, :spend_time + + def total_time_spent + timelogs.sum(:time_spent) + end + + def human_total_time_spent + Gitlab::TimeTrackingFormatter.output(total_time_spent) + end + + def human_time_estimate + Gitlab::TimeTrackingFormatter.output(time_estimate) + end + + private + + def reset_spent_time + timelogs.new(time_spent: total_time_spent * -1, user: @time_spent_user) + end + + def add_or_subtract_spent_time + timelogs.new(time_spent: time_spent, user: @time_spent_user) + end + + def check_negative_time_spent + return if time_spent.nil? || time_spent == :reset + + # we need to cache the total time spent so multiple calls to #valid? + # doesn't give a false error + @original_total_time_spent ||= total_time_spent + + if time_spent < 0 && (time_spent.abs > @original_total_time_spent) + errors.add(:time_spent, 'Time to subtract exceeds the total time spent') + end + end +end diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 24c7b26d223..1ca7f91dc03 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -4,17 +4,21 @@ module TokenAuthenticatable private def write_new_token(token_field) - new_token = generate_token(token_field) + new_token = generate_available_token(token_field) write_attribute(token_field, new_token) end - def generate_token(token_field) + def generate_available_token(token_field) loop do - token = Devise.friendly_token + token = generate_token(token_field) break token unless self.class.unscoped.find_by(token_field => token) end end + def generate_token(token_field) + Devise.friendly_token + end + class_methods do def authentication_token_fields @token_fields || [] @@ -35,6 +39,10 @@ module TokenAuthenticatable current_token.blank? ? write_new_token(token_field) : current_token end + define_method("set_#{token_field}") do |token| + write_attribute(token_field, token) if token + end + define_method("ensure_#{token_field}!") do send("reset_#{token_field}!") if read_attribute(token_field).blank? read_attribute(token_field) diff --git a/app/models/concerns/valid_attribute.rb b/app/models/concerns/valid_attribute.rb new file mode 100644 index 00000000000..8c35cea8d58 --- /dev/null +++ b/app/models/concerns/valid_attribute.rb @@ -0,0 +1,10 @@ +module ValidAttribute + extend ActiveSupport::Concern + + # Checks whether an attribute has failed validation or not + # + # +attribute+ The symbolised name of the attribute i.e :name + def valid_attribute?(attribute) + self.errors.empty? || self.errors.messages[attribute].nil? + end +end diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb index 8ed4a56b19b..d2e626c22e8 100644 --- a/app/models/cycle_analytics.rb +++ b/app/models/cycle_analytics.rb @@ -1,103 +1,38 @@ class CycleAnalytics - include Gitlab::Database::Median - include Gitlab::Database::DateTime + STAGES = %i[issue plan code test review staging production].freeze - DEPLOYMENT_METRIC_STAGES = %i[production staging] - - def initialize(project, from:) + def initialize(project, options) @project = project - @from = from + @options = options end def summary - @summary ||= Summary.new(@project, from: @from) - end - - def issue - calculate_metric(:issue, - Issue.arel_table[:created_at], - [Issue::Metrics.arel_table[:first_associated_with_milestone_at], - Issue::Metrics.arel_table[:first_added_to_board_at]]) - end - - def plan - calculate_metric(:plan, - [Issue::Metrics.arel_table[:first_associated_with_milestone_at], - Issue::Metrics.arel_table[:first_added_to_board_at]], - Issue::Metrics.arel_table[:first_mentioned_in_commit_at]) - end - - def code - calculate_metric(:code, - Issue::Metrics.arel_table[:first_mentioned_in_commit_at], - MergeRequest.arel_table[:created_at]) + @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new(@project, + from: @options[:from], + current_user: @options[:current_user]).data end - def test - calculate_metric(:test, - MergeRequest::Metrics.arel_table[:latest_build_started_at], - MergeRequest::Metrics.arel_table[:latest_build_finished_at]) + def stats + @stats ||= stats_per_stage end - def review - calculate_metric(:review, - MergeRequest.arel_table[:created_at], - MergeRequest::Metrics.arel_table[:merged_at]) + def no_stats? + stats.all? { |hash| hash[:value].nil? } end - def staging - calculate_metric(:staging, - MergeRequest::Metrics.arel_table[:merged_at], - MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + def permissions(user:) + Gitlab::CycleAnalytics::Permissions.get(user: user, project: @project) end - def production - calculate_metric(:production, - Issue.arel_table[:created_at], - MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + def [](stage_name) + Gitlab::CycleAnalytics::Stage[stage_name].new(project: @project, options: @options) end private - def calculate_metric(name, start_time_attrs, end_time_attrs) - cte_table = Arel::Table.new("cte_table_for_#{name}") - - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new( - cte_table, - subtract_datetimes(base_query_for(name), end_time_attrs, start_time_attrs, name.to_s)) - - median_datetime(cte_table, interval_query, name) - end - - # Join table with a row for every <issue,merge_request> pair (where the merge request - # closes the given issue) with issue and merge request metrics included. The metrics - # are loaded with an inner join, so issues / merge requests without metrics are - # automatically excluded. - def base_query_for(name) - arel_table = MergeRequestsClosingIssues.arel_table - - # Load issues - query = arel_table.join(Issue.arel_table).on(Issue.arel_table[:id].eq(arel_table[:issue_id])). - join(Issue::Metrics.arel_table).on(Issue.arel_table[:id].eq(Issue::Metrics.arel_table[:issue_id])). - where(Issue.arel_table[:project_id].eq(@project.id)). - where(Issue.arel_table[:deleted_at].eq(nil)). - where(Issue.arel_table[:created_at].gteq(@from)) - - # Load merge_requests - query = query.join(MergeRequest.arel_table, Arel::Nodes::OuterJoin). - on(MergeRequest.arel_table[:id].eq(arel_table[:merge_request_id])). - join(MergeRequest::Metrics.arel_table). - on(MergeRequest.arel_table[:id].eq(MergeRequest::Metrics.arel_table[:merge_request_id])) - - if DEPLOYMENT_METRIC_STAGES.include?(name) - # Limit to merge requests that have been deployed to production after `@from` - query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from)) + def stats_per_stage + STAGES.map do |stage_name| + self[stage_name].as_json end - - query end end diff --git a/app/models/cycle_analytics/summary.rb b/app/models/cycle_analytics/summary.rb index b46db449bf3..e69de29bb2d 100644 --- a/app/models/cycle_analytics/summary.rb +++ b/app/models/cycle_analytics/summary.rb @@ -1,42 +0,0 @@ -class CycleAnalytics - class Summary - def initialize(project, from:) - @project = project - @from = from - end - - def new_issues - @project.issues.created_after(@from).count - end - - def commits - ref = @project.default_branch.presence - count_commits_for(ref) - end - - def deploys - @project.deployments.where("created_at > ?", @from).count - end - - private - - # Don't use the `Gitlab::Git::Repository#log` method, because it enforces - # a limit. Since we need a commit count, we _can't_ enforce a limit, so - # the easiest way forward is to replicate the relevant portions of the - # `log` function here. - def count_commits_for(ref) - return unless ref - - repository = @project.repository.raw_repository - sha = @project.repository.commit(ref).sha - - cmd = %W(git --git-dir=#{repository.path} log) - cmd << '--format=%H' - cmd << "--after=#{@from.iso8601}" - cmd << sha - - raw_output = IO.popen(cmd) { |io| io.read } - raw_output.lines.count - end - end -end diff --git a/app/models/dashboard_milestone.rb b/app/models/dashboard_milestone.rb new file mode 100644 index 00000000000..646c1e5ce1a --- /dev/null +++ b/app/models/dashboard_milestone.rb @@ -0,0 +1,5 @@ +class DashboardMilestone < GlobalMilestone + def issues_finder_params + { authorized_only: true } + end +end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 2c525d4cd7a..053f2a11aa0 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -20,4 +20,18 @@ class DeployKey < Key def destroyed_when_orphaned? self.private? end + + def has_access_to?(project) + projects.include?(project) + end + + def can_push_to?(project) + can_push? && has_access_to?(project) + end + + private + + # we don't want to notify the user for deploy keys + def notify_user + end end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index de06c13481a..bbe813db823 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -25,7 +25,12 @@ class Discussion to: :last_resolved_note, allow_nil: true - delegate :blob, :highlighted_diff_lines, to: :diff_file, allow_nil: true + delegate :blob, + :highlighted_diff_lines, + :diff_lines, + + to: :diff_file, + allow_nil: true def self.for_notes(notes) notes.group_by(&:discussion_id).values.map { |notes| new(notes) } @@ -83,6 +88,10 @@ class Discussion @first_note ||= @notes.first end + def first_note_to_resolve + @first_note_to_resolve ||= notes.detect(&:to_be_resolved?) + end + def last_note @last_note ||= @notes.last end @@ -159,10 +168,11 @@ class Discussion end # Returns an array of at most 16 highlighted lines above a diff note - def truncated_diff_lines + def truncated_diff_lines(highlight: true) + lines = highlight ? highlighted_diff_lines : diff_lines prev_lines = [] - highlighted_diff_lines.each do |line| + lines.each do |line| if line.meta? prev_lines.clear else diff --git a/app/models/environment.rb b/app/models/environment.rb index 73f415c0ef0..652abf18a8a 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -1,25 +1,38 @@ class Environment < ActiveRecord::Base + # Used to generate random suffixes for the slug + NUMBERS = '0'..'9' + SUFFIX_CHARS = ('a'..'z').to_a + NUMBERS.to_a + belongs_to :project, required: true, validate: true has_many :deployments before_validation :nullify_external_url + before_validation :generate_slug, if: ->(env) { env.slug.blank? } + before_save :set_environment_type validates :name, presence: true, uniqueness: { scope: :project_id }, - length: { within: 0..255 }, + length: { maximum: 255 }, format: { with: Gitlab::Regex.environment_name_regex, message: Gitlab::Regex.environment_name_regex_message } + validates :slug, + presence: true, + uniqueness: { scope: :project_id }, + length: { maximum: 24 }, + format: { with: Gitlab::Regex.environment_slug_regex, + message: Gitlab::Regex.environment_slug_regex_message } + validates :external_url, uniqueness: { scope: :project_id }, length: { maximum: 255 }, allow_nil: true, addressable_url: true - delegate :stop_action, to: :last_deployment, allow_nil: true + delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true scope :available, -> { with_state(:available) } scope :stopped, -> { with_state(:stopped) } @@ -37,6 +50,17 @@ class Environment < ActiveRecord::Base state :stopped end + def predefined_variables + [ + { key: 'CI_ENVIRONMENT_NAME', value: name, public: true }, + { key: 'CI_ENVIRONMENT_SLUG', value: slug, public: true }, + ] + end + + def recently_updated_on_branch?(ref) + ref.to_s == last_deployment.try(:ref) + end + def last_deployment deployments.last end @@ -63,7 +87,7 @@ class Environment < ActiveRecord::Base end def update_merge_request_metrics? - self.name == "production" + (environment_type || name) == "production" end def first_deployment_for(commit) @@ -92,6 +116,60 @@ class Environment < ActiveRecord::Base def stop!(current_user) return unless stoppable? + stop stop_action.play(current_user) end + + def actions_for(environment) + return [] unless manual_actions + + manual_actions.select do |action| + action.expanded_environment_name == environment + end + end + + def has_terminals? + project.deployment_service.present? && available? && last_deployment.present? + end + + def terminals + project.deployment_service.terminals(self) if has_terminals? + end + + # An environment name is not necessarily suitable for use in URLs, DNS + # or other third-party contexts, so provide a slugified version. A slug has + # the following properties: + # * contains only lowercase letters (a-z), numbers (0-9), and '-' + # * begins with a letter + # * has a maximum length of 24 bytes (OpenShift limitation) + # * cannot end with `-` + def generate_slug + # Lowercase letters and numbers only + slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') + + # Must start with a letter + slugified = "env-" + slugified if NUMBERS.cover?(slugified[0]) + + # Maximum length: 24 characters (OpenShift limitation) + slugified = slugified[0..23] + + # Cannot end with a "-" character (Kubernetes label limitation) + slugified = slugified[0..-2] if slugified[-1] == "-" + + # Add a random suffix, shortening the current string if necessary, if it + # has been slugified. This ensures uniqueness. + slugified = slugified[0..16] + "-" + random_suffix if slugified != name + + self.slug = slugified + end + + private + + # Slugifying a name may remove the uniqueness guarantee afforded by it being + # based on name (which must be unique). To compensate, we add a random + # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, + # but the chance of collisions is vanishingly small + def random_suffix + (0..5).map { SUFFIX_CHARS.sample }.join + end end diff --git a/app/models/event.rb b/app/models/event.rb index 43e67069b70..2662f170765 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -43,12 +43,7 @@ class Event < ActiveRecord::Base scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) } class << self - def reset_event_cache_for(target) - Event.where(target_id: target.id, target_type: target.class.to_s). - order('id DESC').limit(100). - update_all(updated_at: Time.now) - end - + # Update Gitlab::ContributionsCalendar#activity_dates if this changes def contributions where("action = ? OR (target_type in (?) AND action in (?))", Event::PUSHED, ["MergeRequest", "Issue"], @@ -61,8 +56,8 @@ class Event < ActiveRecord::Base end def visible_to_user?(user = nil) - if push? - true + if push? || commit_note? + Ability.allowed?(user, :download_code, project) elsif membership_changed? true elsif created_project? @@ -282,7 +277,7 @@ class Event < ActiveRecord::Base end def commit_note? - target.for_commit? + note? && target && target.for_commit? end def issue_note? @@ -294,7 +289,7 @@ class Event < ActiveRecord::Base end def project_snippet_note? - target.for_snippet? + note? && target && target.for_snippet? end def note_target @@ -352,6 +347,10 @@ class Event < ActiveRecord::Base update_all(last_activity_at: created_at) end + def authored_by?(user) + user ? author_id == user.id : false + end + private def recent_update? diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index fd9a8c1b8b7..26712c19b5a 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -29,7 +29,16 @@ class ExternalIssue @project end - def to_reference(_from_project = nil) + def project_id + @project.id + end + + # Pattern used to extract `JIRA-123` issue references from text + def self.reference_pattern + @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} + end + + def to_reference(_from_project = nil, full: nil) id end diff --git a/app/models/forked_project_link.rb b/app/models/forked_project_link.rb index 9803bae0bee..36cf7ad6a28 100644 --- a/app/models/forked_project_link.rb +++ b/app/models/forked_project_link.rb @@ -1,4 +1,4 @@ class ForkedProjectLink < ActiveRecord::Base - belongs_to :forked_to_project, class_name: Project - belongs_to :forked_from_project, class_name: Project + belongs_to :forked_to_project, class_name: 'Project' + belongs_to :forked_from_project, class_name: 'Project' end diff --git a/app/models/generic_commit_status.rb b/app/models/generic_commit_status.rb index fa54e3540d0..8867ba0d2ff 100644 --- a/app/models/generic_commit_status.rb +++ b/app/models/generic_commit_status.rb @@ -1,6 +1,10 @@ class GenericCommitStatus < CommitStatus before_validation :set_default_values + validates :target_url, addressable_url: true, + length: { maximum: 255 }, + allow_nil: true + # GitHub compatible API alias_attribute :context, :name @@ -12,4 +16,10 @@ class GenericCommitStatus < CommitStatus def tags [:external] end + + def detailed_status(current_user) + Gitlab::Ci::Status::External::Factory + .new(self, current_user) + .fabricate! + end end diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index cde4a568577..b991d78e27f 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -1,6 +1,8 @@ class GlobalMilestone include Milestoneish + EPOCH = DateTime.parse('1970-01-01') + attr_accessor :title, :milestones alias_attribute :name, :title @@ -8,13 +10,22 @@ class GlobalMilestone @first_milestone end - def self.build_collection(milestones) - milestones = milestones.group_by(&:title) + def self.build_collection(projects, params) + child_milestones = MilestonesFinder.new.execute(projects, params) - milestones.map do |title, milestones| - milestones_relation = Milestone.where(id: milestones.map(&:id)) + milestones = child_milestones.select(:id, :title).group_by(&:title).map do |title, grouped| + milestones_relation = Milestone.where(id: grouped.map(&:id)) new(title, milestones_relation) end + + milestones.sort_by { |milestone| milestone.due_date || EPOCH } + end + + def self.build(projects, title) + child_milestones = Milestone.of_projects(projects).where(title: title) + return if child_milestones.blank? + + new(title, child_milestones) end def initialize(title, milestones) @@ -24,30 +35,24 @@ class GlobalMilestone @first_milestone = milestones.find {|m| m.description.present? } || milestones.first end - def safe_title - @title.to_slug.normalize.to_s + def milestoneish_ids + milestones.select(:id) end - def expired? - if due_date - due_date.past? - else - false - end + def safe_title + @title.to_slug.normalize.to_s end def projects - @projects ||= Project.for_milestones(milestones.select(:id)) + @projects ||= Project.for_milestones(milestoneish_ids) end def state - state = milestones.map { |milestone| milestone.state } - - if state.count('closed') == state.size - 'closed' - else - 'active' + milestones.each do |milestone| + return 'active' if milestone.state != 'closed' end + + 'closed' end def active? @@ -59,11 +64,11 @@ class GlobalMilestone end def issues - @issues ||= Issue.of_milestones(milestones.select(:id)).includes(:project, :assignee, :labels) + @issues ||= Issue.of_milestones(milestoneish_ids).includes(:project, :assignee, :labels) end def merge_requests - @merge_requests ||= MergeRequest.of_milestones(milestones.select(:id)).includes(:target_project, :assignee, :labels) + @merge_requests ||= MergeRequest.of_milestones(milestoneish_ids).includes(:target_project, :assignee, :labels) end def participants @@ -81,18 +86,15 @@ class GlobalMilestone @due_date = if @milestones.all? { |x| x.due_date == @milestones.first.due_date } @milestones.first.due_date - else - nil end end - def expires_at - if due_date - if due_date.past? - "expired on #{due_date.to_s(:medium)}" - else - "expires on #{due_date.to_s(:medium)}" + def start_date + return @start_date if defined?(@start_date) + + @start_date = + if @milestones.all? { |x| x.start_date == @milestones.first.start_date } + @milestones.first.start_date end - end end end diff --git a/app/models/group.rb b/app/models/group.rb index d9e90cd256a..99675ddb366 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -5,6 +5,7 @@ class Group < Namespace include Gitlab::VisibilityLevel include AccessRequestable include Referable + include SelectForProjectAuthorization has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :group_members @@ -47,7 +48,13 @@ class Group < Namespace end def sort(method) - order_by(method) + if method == 'storage_size_desc' + # storage_size is a virtual column so we need to + # pass a string to avoid AR adding the table name + reorder('storage_size DESC, namespaces.id DESC') + else + order_by(method) + end end def reference_prefix @@ -61,9 +68,19 @@ class Group < Namespace def visible_to_user(user) where(id: user.authorized_groups.select(:id).reorder(nil)) end + + def select_for_project_authorization + if current_scope.joins_values.include?(:shared_projects) + joins('INNER JOIN namespaces project_namespace ON project_namespace.id = projects.namespace_id') + .where('project_namespace.share_with_group_lock = ?', false) + .select("members.user_id, projects.id AS project_id, LEAST(project_group_links.group_access, members.access_level) AS access_level") + else + super + end + end end - def to_reference(_from_project = nil) + def to_reference(_from_project = nil, full: nil) "#{self.class.reference_prefix}#{name}" end @@ -72,7 +89,7 @@ class Group < Namespace end def human_name - name + full_name end def visibility_level_field @@ -144,15 +161,17 @@ class Group < Namespace end def has_owner?(user) - owners.include?(user) + members_with_parents.owners.where(user_id: user).any? end def has_master?(user) - members.masters.where(user_id: user).any? + members_with_parents.masters.where(user_id: user).any? end + # Check if user is a last owner of the group. + # Parent owners are ignored for nested groups. def last_owner?(user) - has_owner?(user) && owners.size == 1 + owners.include?(user) && owners.size == 1 end def avatar_type @@ -176,4 +195,16 @@ class Group < Namespace def system_hook_service SystemHooksService.new end + + def refresh_members_authorized_projects + UserProjectAccessChangedService.new(users_with_parents.pluck(:id)).execute + end + + def members_with_parents + GroupMember.where(requested_at: nil, source_id: parents.map(&:id).push(id)) + end + + def users_with_parents + User.where(id: members_with_parents.select(:user_id)) + end end diff --git a/app/models/group_label.rb b/app/models/group_label.rb index 68841ace2e6..92c83b54861 100644 --- a/app/models/group_label.rb +++ b/app/models/group_label.rb @@ -8,8 +8,4 @@ class GroupLabel < Label def subject_foreign_key 'group_id' end - - def to_reference(source_project = nil, target_project = nil, format: :id) - super(source_project, target_project, format: format) - end end diff --git a/app/models/group_milestone.rb b/app/models/group_milestone.rb new file mode 100644 index 00000000000..7b6db2634b7 --- /dev/null +++ b/app/models/group_milestone.rb @@ -0,0 +1,19 @@ +class GroupMilestone < GlobalMilestone + attr_accessor :group + + def self.build_collection(group, projects, params) + super(projects, params).each do |milestone| + milestone.group = group + end + end + + def self.build(group, projects, title) + super(projects, title).tap do |milestone| + milestone.group = group if milestone + end + end + + def issues_finder_params + { group_id: group.id } + end +end diff --git a/app/models/guest.rb b/app/models/guest.rb new file mode 100644 index 00000000000..01285ca1264 --- /dev/null +++ b/app/models/guest.rb @@ -0,0 +1,7 @@ +class Guest + class << self + def can?(action, subject) + Ability.allowed?(nil, action, subject) + end + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 4f02b02c488..65638d9a299 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -39,6 +39,8 @@ class Issue < ActiveRecord::Base scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } + scope :include_associations, -> { includes(:assignee, :labels, project: :namespace) } + attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -60,61 +62,6 @@ class Issue < ActiveRecord::Base attributes end - class << self - private - - # Returns the project that the current scope belongs to if any, nil otherwise. - # - # Examples: - # - my_project.issues.without_due_date.owner_project => my_project - # - Issue.all.owner_project => nil - def owner_project - # No owner if we're not being called from an association - return unless all.respond_to?(:proxy_association) - - owner = all.proxy_association.owner - - # Check if the association is or belongs to a project - if owner.is_a?(Project) - owner - else - begin - owner.association(:project).target - rescue ActiveRecord::AssociationNotFoundError - nil - end - end - end - end - - def self.visible_to_user(user) - return where('issues.confidential IS NULL OR issues.confidential IS FALSE') if user.blank? - return all if user.admin? - - # Check if we are scoped to a specific project's issues - if owner_project - if owner_project.authorized_for_user?(user, Gitlab::Access::REPORTER) - # If the project is authorized for the user, they can see all issues in the project - return all - else - # else only non confidential and authored/assigned to them - return where('issues.confidential IS NULL OR issues.confidential IS FALSE - OR issues.author_id = :user_id OR issues.assignee_id = :user_id', - user_id: user.id) - end - end - - where(' - issues.confidential IS NULL - OR issues.confidential IS FALSE - OR (issues.confidential = TRUE - AND (issues.author_id = :user_id - OR issues.assignee_id = :user_id - OR issues.project_id IN(:project_ids)))', - user_id: user.id, - project_ids: user.authorized_projects(Gitlab::Access::REPORTER).select(:id)) - end - def self.reference_prefix '#' end @@ -150,14 +97,10 @@ class Issue < ActiveRecord::Base end end - def to_reference(from_project = nil) + def to_reference(from_project = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" - if cross_project_reference?(from_project) - reference = project.to_reference + reference - end - - reference + "#{project.to_reference(from_project, full: full)}#{reference}" end def referenced_merge_requests(current_user = nil) @@ -182,18 +125,6 @@ class Issue < ActiveRecord::Base branches_with_iid - branches_with_merge_request end - # Reset issue events cache - # - # Since we do cache @event we need to reset cache in special cases: - # * when an issue is updated - # Events cache stored like events/23-20130109142513. - # The cache key includes updated_at timestamp. - # Thus it will automatically generate a new fragment - # when the event is updated because the key changes. - def reset_events_cache - Event.reset_event_cache_for(self) - end - # To allow polymorphism with MergeRequest. def source_project project @@ -250,29 +181,9 @@ class Issue < ActiveRecord::Base # Returns `true` if the current issue can be viewed by either a logged in User # or an anonymous user. def visible_to_user?(user = nil) - user ? readable_by?(user) : publicly_visible? - end - - # Returns `true` if the given User can read the current Issue. - def readable_by?(user) - if user.admin? - true - elsif project.owner == user - true - elsif confidential? - author == user || - assignee == user || - project.team.member?(user, Gitlab::Access::REPORTER) - else - project.public? || - project.internal? && !user.external? || - project.team.member?(user) - end - end + return false unless project.feature_available?(:issues, user) - # Returns `true` if this Issue is visible to everybody. - def publicly_visible? - project.public? && !confidential? + user ? readable_by?(user) : publicly_visible? end def overdue? @@ -286,7 +197,7 @@ class Issue < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| - json[:subscribed] = subscribed?(options[:user]) if options.has_key?(:user) + json[:subscribed] = subscribed?(options[:user], project) if options.has_key?(:user) && options[:user] if options.has_key?(:labels) json[:labels] = labels.as_json( @@ -297,4 +208,32 @@ class Issue < ActiveRecord::Base end end end + + private + + # Returns `true` if the given User can read the current Issue. + # + # This method duplicates the same check of issue_policy.rb + # for performance reasons, check commit: 002ad215818450d2cbbc5fa065850a953dc7ada8 + # Make sure to sync this method with issue_policy.rb + def readable_by?(user) + if user.admin? + true + elsif project.owner == user + true + elsif confidential? + author == user || + assignee == user || + project.team.member?(user, Gitlab::Access::REPORTER) + else + project.public? || + project.internal? && !user.external? || + project.team.member?(user) + end + end + + # Returns `true` if this Issue is visible to everybody. + def publicly_visible? + project.public? && !confidential? + end end diff --git a/app/models/issue_collection.rb b/app/models/issue_collection.rb new file mode 100644 index 00000000000..f0b7d9914c8 --- /dev/null +++ b/app/models/issue_collection.rb @@ -0,0 +1,42 @@ +# IssueCollection can be used to reduce a list of issues down to a subset. +# +# IssueCollection is not meant to be some sort of Enumerable, instead it's meant +# to take a list of issues and return a new list of issues based on some +# criteria. For example, given a list of issues you may want to return a list of +# issues that can be read or updated by a given user. +class IssueCollection + attr_reader :collection + + def initialize(collection) + @collection = collection + end + + # Returns all the issues that can be updated by the user. + def updatable_by_user(user) + return collection if user.admin? + + # Given all the issue projects we get a list of projects that the current + # user has at least reporter access to. + projects_with_reporter_access = user. + projects_with_reporter_access_limited_to(project_ids). + pluck(:id) + + collection.select do |issue| + if projects_with_reporter_access.include?(issue.project_id) + true + elsif issue.is_a?(Issue) + issue.assignee_or_author?(user) + else + false + end + end + end + + alias_method :visible_to, :updatable_by_user + + private + + def project_ids + @project_ids ||= collection.map(&:project_id).uniq + end +end diff --git a/app/models/key.rb b/app/models/key.rb index 568a60b8af3..9c74ca84753 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -4,14 +4,24 @@ class Key < ActiveRecord::Base include AfterCommitQueue include Sortable - belongs_to :user + LAST_USED_AT_REFRESH_TIME = 1.day.to_i - before_validation :strip_white_space, :generate_fingerprint + belongs_to :user - validates :title, presence: true, length: { within: 0..255 } - validates :key, presence: true, length: { within: 0..5000 }, format: { with: /\A(ssh|ecdsa)-.*\Z/ } - validates :key, format: { without: /\n|\r/, message: 'should be a single line' } - validates :fingerprint, uniqueness: true, presence: { message: 'cannot be generated' } + before_validation :generate_fingerprint + + validates :title, + presence: true, + length: { maximum: 255 } + validates :key, + presence: true, + length: { maximum: 5000 }, + format: { with: /\A(ssh|ecdsa)-.*\Z/ } + validates :key, + format: { without: /\n|\r/, message: 'should be a single line' } + validates :fingerprint, + uniqueness: true, + presence: { message: 'cannot be generated' } delegate :name, :email, to: :user, prefix: true @@ -21,8 +31,9 @@ class Key < ActiveRecord::Base after_destroy :remove_from_shell after_destroy :post_destroy_hook - def strip_white_space - self.key = key.strip unless key.blank? + def key=(value) + value.strip! unless value.blank? + write_attribute(:key, value) end def publishable_key @@ -40,6 +51,13 @@ class Key < ActiveRecord::Base "key-#{id}" end + def update_last_used_at + lease = Gitlab::ExclusiveLease.new("key_update_last_used_at:#{id}", timeout: LAST_USED_AT_REFRESH_TIME) + return unless lease.try_obtain + + UseKeyWorker.perform_async(id) + end + def add_to_shell GitlabShellWorker.perform_async( :add_key, @@ -48,10 +66,6 @@ class Key < ActiveRecord::Base ) end - def notify_user - run_after_commit { NotificationService.new.new_key(self) } - end - def post_create_hook SystemHooksService.new.execute_hooks_for(self, :create) end @@ -77,4 +91,8 @@ class Key < ActiveRecord::Base self.fingerprint = Gitlab::KeyFingerprint.new(self.key).fingerprint end + + def notify_user + run_after_commit { NotificationService.new.new_key(self) } + end end diff --git a/app/models/label.rb b/app/models/label.rb index d9287f2dc29..5b6b9a7a736 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -26,6 +26,7 @@ class Label < ActiveRecord::Base # Don't allow ',' for label titles validates :title, presence: true, format: { with: /\A[^,]+\z/ } validates :title, uniqueness: { scope: [:group_id, :project_id] } + validates :title, length: { maximum: 255 } default_scope { order(title: :asc) } @@ -144,18 +145,19 @@ class Label < ActiveRecord::Base # # Examples: # - # Label.first.to_reference # => "~1" - # Label.first.to_reference(format: :name) # => "~\"bug\"" - # Label.first.to_reference(project1, project2) # => "gitlab-org/gitlab-ce~1" + # Label.first.to_reference # => "~1" + # Label.first.to_reference(format: :name) # => "~\"bug\"" + # Label.first.to_reference(project, target_project: same_namespace_project) # => "gitlab-ce~1" + # Label.first.to_reference(project, target_project: another_namespace_project) # => "gitlab-org/gitlab-ce~1" # # Returns a String # - def to_reference(source_project = nil, target_project = nil, format: :id) + def to_reference(from_project = nil, target_project: nil, format: :id, full: false) format_reference = label_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if cross_project_reference?(source_project, target_project) - source_project.to_reference + reference + if from_project + "#{from_project.to_reference(target_project, full: full)}#{reference}" else reference end @@ -169,10 +171,6 @@ class Label < ActiveRecord::Base private - def cross_project_reference?(source_project, target_project) - source_project && target_project && source_project != target_project - end - def issues_count(user, params = {}) params.merge!(subject_foreign_key => subject.id, label_name: title, scope: 'all') IssuesFinder.new(user, params.with_indifferent_access).execute.count diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb index 0fd5f089db9..007eed5600a 100644 --- a/app/models/lfs_objects_project.rb +++ b/app/models/lfs_objects_project.rb @@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base validates :lfs_object_id, presence: true validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" } validates :project_id, presence: true + + after_create :update_project_statistics + after_destroy :update_project_statistics + + private + + def update_project_statistics + ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size]) + end end diff --git a/app/models/member.rb b/app/models/member.rb index b89ba8ecbb8..c585e0b450e 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -57,12 +57,18 @@ class Member < ActiveRecord::Base scope :owners, -> { active.where(access_level: OWNER) } scope :owners_and_masters, -> { active.where(access_level: [OWNER, MASTER]) } + scope :order_name_asc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'ASC')) } + scope :order_name_desc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'DESC')) } + scope :order_recent_sign_in, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.last_sign_in_at', 'DESC')) } + scope :order_oldest_sign_in, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.last_sign_in_at', 'ASC')) } + before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? } after_create :send_invite, if: :invite?, unless: :importing? after_create :send_request, if: :request?, unless: :importing? after_create :create_notification_setting, unless: [:pending?, :importing?] after_create :post_create_hook, unless: [:pending?, :importing?] + after_create :refresh_member_authorized_projects, if: :importing? after_update :post_update_hook, unless: [:pending?, :importing?] after_destroy :post_destroy_hook, unless: :pending? @@ -71,6 +77,34 @@ class Member < ActiveRecord::Base default_value_for :notification_level, NotificationSetting.levels[:global] class << self + def search(query) + joins(:user).merge(User.search(query)) + end + + def sort(method) + case method.to_s + when 'access_level_asc' then reorder(access_level: :asc) + when 'access_level_desc' then reorder(access_level: :desc) + when 'recent_sign_in' then order_recent_sign_in + when 'oldest_sign_in' then order_oldest_sign_in + when 'last_joined' then order_created_desc + when 'oldest_joined' then order_created_asc + else + order_by(method) + end + end + + def left_join_users + users = User.arel_table + members = Member.arel_table + + member_users = members.join(users, Arel::Nodes::OuterJoin). + on(members[:user_id].eq(users[:id])). + join_sources + + joins(member_users) + end + def access_for_user_ids(user_ids) where(user_id: user_ids).has_access.pluck(:user_id, :access_level).to_h end @@ -88,8 +122,8 @@ class Member < ActiveRecord::Base member = if user.is_a?(User) source.members.find_by(user_id: user.id) || - source.requesters.find_by(user_id: user.id) || - source.members.build(user_id: user.id) + source.requesters.find_by(user_id: user.id) || + source.members.build(user_id: user.id) else source.members.build(invite_email: user) end @@ -113,6 +147,8 @@ class Member < ActiveRecord::Base member.save end + UserProjectAccessChangedService.new(user.id).execute if user.is_a?(User) + member end @@ -239,17 +275,28 @@ class Member < ActiveRecord::Base end def post_create_hook + UserProjectAccessChangedService.new(user.id).execute system_hook_service.execute_hooks_for(self, :create) end def post_update_hook - # override in subclass + UserProjectAccessChangedService.new(user.id).execute if access_level_changed? end def post_destroy_hook + refresh_member_authorized_projects system_hook_service.execute_hooks_for(self, :destroy) end + def refresh_member_authorized_projects + # If user/source is being destroyed, project access are gonna be destroyed eventually + # because of DB foreign keys, so we shouldn't bother with refreshing after each + # member is destroyed through association + return if destroyed_by_association.present? + + UserProjectAccessChangedService.new(user_id).execute + end + def after_accept_invite post_create_hook end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 0397c57f935..cd5b345bae5 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -22,7 +22,8 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :reload_diff_if_branch_changed - delegate :commits, :real_size, to: :merge_request_diff, prefix: nil + delegate :commits, :real_size, :commits_sha, :commits_count, + to: :merge_request_diff, prefix: nil # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -90,17 +91,23 @@ class MergeRequest < ActiveRecord::Base around_transition do |merge_request, transition, block| Gitlab::Timeless.timeless(merge_request, &block) end + + after_transition unchecked: :cannot_be_merged do |merge_request, transition| + TodoService.new.merge_request_became_unmergeable(merge_request) + end end validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_without_fork?] validates :source_branch, presence: true validates :target_project, presence: true validates :target_branch, presence: true - validates :merge_user, presence: true, if: :merge_when_build_succeeds? + validates :merge_user, presence: true, if: :merge_when_build_succeeds?, unless: :importing? validate :validate_branches, unless: [:allow_broken, :importing?, :closed_without_fork?] validate :validate_fork, unless: :closed_without_fork? - scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) } + scope :by_source_or_target_branch, ->(branch_name) do + where("source_branch = :branch OR target_branch = :branch", branch: branch_name) + end scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) } scope :by_milestone, ->(milestone) { where(milestone_id: milestone) } scope :of_projects, ->(ids) { where(target_project_id: ids) } @@ -172,14 +179,10 @@ class MergeRequest < ActiveRecord::Base work_in_progress?(title) ? title : "WIP: #{title}" end - def to_reference(from_project = nil) + def to_reference(from_project = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" - if cross_project_reference?(from_project) - reference = project.to_reference + reference - end - - reference + "#{project.to_reference(from_project, full: full)}#{reference}" end def first_commit @@ -199,7 +202,9 @@ class MergeRequest < ActiveRecord::Base end def diff_size - diffs(diff_options).size + opts = diff_options || {} + + raw_diffs(opts).size end def diff_base_commit @@ -220,7 +225,7 @@ class MergeRequest < ActiveRecord::Base # true base commit, so we can't simply have `#diff_base_commit` fall back on # this method. def likely_diff_base_commit - first_commit.parent || first_commit + first_commit.try(:parent) || first_commit end def diff_start_commit @@ -425,6 +430,7 @@ class MergeRequest < ActiveRecord::Base return false if work_in_progress? return false if broken? return false unless skip_ci_check || mergeable_ci_state? + return false unless mergeable_discussions_state? true end @@ -441,18 +447,18 @@ class MergeRequest < ActiveRecord::Base end def should_remove_source_branch? - merge_params['should_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['should_remove_source_branch']) end def force_remove_source_branch? - merge_params['force_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['force_remove_source_branch']) end def remove_source_branch? should_remove_source_branch? || force_remove_source_branch? end - def mr_and_commit_notes + def related_notes # Fetch comments only from last 100 commits commits_for_notes_limit = 100 commit_ids = commits.last(commits_for_notes_limit).map(&:id) @@ -468,7 +474,7 @@ class MergeRequest < ActiveRecord::Base end def discussions - @discussions ||= self.mr_and_commit_notes. + @discussions ||= self.related_notes. inc_relations_for_view. fresh. discussions @@ -478,6 +484,14 @@ class MergeRequest < ActiveRecord::Base @diff_discussions ||= self.notes.diff_notes.discussions end + def resolvable_discussions + @resolvable_discussions ||= diff_discussions.select(&:to_be_resolved?) + end + + def discussions_can_be_resolved_by?(user) + resolvable_discussions.all? { |discussion| discussion.can_resolve?(user) } + end + def find_diff_discussion(discussion_id) notes = self.notes.diff_notes.where(discussion_id: discussion_id).fresh.to_a return if notes.empty? @@ -493,6 +507,16 @@ class MergeRequest < ActiveRecord::Base discussions_resolvable? && diff_discussions.none?(&:to_be_resolved?) end + def discussions_to_be_resolved? + discussions_resolvable? && !discussions_resolved? + end + + def mergeable_discussions_state? + return true unless project.only_allow_merge_if_all_discussions_are_resolved? + + !discussions_to_be_resolved? + end + def hook_attrs attrs = { source: source_project.try(:hook_attrs), @@ -550,6 +574,15 @@ class MergeRequest < ActiveRecord::Base end end + def issues_mentioned_but_not_closing(current_user = self.author) + return [] unless target_branch == project.default_branch + + ext = Gitlab::ReferenceExtractor.new(project, current_user) + ext.analyze(description) + + ext.issues - closes_issues + end + def target_project_path if target_project target_project.path_with_namespace @@ -594,25 +627,24 @@ class MergeRequest < ActiveRecord::Base self.target_project.repository.branch_names.include?(self.target_branch) end - # Reset merge request events cache - # - # Since we do cache @event we need to reset cache in special cases: - # * when a merge request is updated - # Events cache stored like events/23-20130109142513. - # The cache key includes updated_at timestamp. - # Thus it will automatically generate a new fragment - # when the event is updated because the key changes. - def reset_events_cache - Event.reset_event_cache_for(self) - end - - def merge_commit_message - message = "Merge branch '#{source_branch}' into '#{target_branch}'\n\n" - message << "#{title}\n\n" - message << "#{description}\n\n" if description.present? + def merge_commit_message(include_description: false) + closes_issues_references = closes_issues.map do |issue| + issue.to_reference(target_project) + end + + message = [ + "Merge branch '#{source_branch}' into '#{target_branch}'", + title + ] + + if !include_description && closes_issues_references.present? + message << "Closes #{closes_issues_references.to_sentence}" + end + + message << "#{description}" if include_description && description.present? message << "See merge request #{to_reference}" - message + message.join("\n\n") end def reset_merge_when_build_succeeds @@ -663,7 +695,7 @@ class MergeRequest < ActiveRecord::Base end def broken? - self.commits.blank? || branch_missing? || cannot_be_merged? + has_no_commits? || branch_missing? || cannot_be_merged? end def can_be_merged_by?(user) @@ -679,18 +711,21 @@ class MergeRequest < ActiveRecord::Base def mergeable_ci_state? return true unless project.only_allow_merge_if_build_succeeds? - !pipeline || pipeline.success? + !head_pipeline || head_pipeline.success? || head_pipeline.skipped? end def environments return [] unless diff_head_commit - @environments ||= - begin - envs = target_project.environments_for(target_branch, diff_head_commit, with_tags: true) - envs.concat(source_project.environments_for(source_branch, diff_head_commit)) if source_project - envs.uniq - end + @environments ||= begin + target_envs = target_project.environments_for( + target_branch, commit: diff_head_commit, with_tags: true) + + source_envs = source_project.environments_for( + source_branch, commit: diff_head_commit) if source_project + + (target_envs.to_a + source_envs.to_a).uniq + end end def state_human_name @@ -768,18 +803,14 @@ class MergeRequest < ActiveRecord::Base diverged_commits_count > 0 end - def commits_sha - commits.map(&:sha) - end - - def pipeline + def head_pipeline return unless diff_head_sha && source_project - @pipeline ||= source_project.pipeline_for(source_branch, diff_head_sha) + @head_pipeline ||= source_project.pipeline_for(source_branch, diff_head_sha) end def all_pipelines - return unless source_project + return Ci::Pipeline.none unless source_project @all_pipelines ||= source_project.pipelines .where(sha: all_commits_sha, ref: source_branch) @@ -802,7 +833,7 @@ class MergeRequest < ActiveRecord::Base @merge_commit ||= project.commit(merge_commit_sha) if merge_commit_sha end - def can_be_reverted?(current_user = nil) + def can_be_reverted?(current_user) merge_commit && !merge_commit.has_been_reverted?(current_user, self) end @@ -869,4 +900,24 @@ class MergeRequest < ActiveRecord::Base @conflicts_can_be_resolved_in_ui = false end end + + def has_commits? + merge_request_diff && commits_count > 0 + end + + def has_no_commits? + !has_commits? + end + + def mergeable_with_slash_command?(current_user, autocomplete_precheck: false, last_diff_sha: nil) + return false unless can_be_merged_by?(current_user) + + return true if autocomplete_precheck + + return false unless mergeable?(skip_ci_check: true) + return false if head_pipeline && !(head_pipeline.success? || head_pipeline.active?) + return false if last_diff_sha != diff_head_sha + + true + end end diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index 99c49a020c9..cdc408738be 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -1,5 +1,6 @@ class MergeRequest::Metrics < ActiveRecord::Base belongs_to :merge_request + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :pipeline_id def record! if merge_request.merged? && self.merged_at.blank? diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index dd65a9a8b86..dadb81f9b6e 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -1,7 +1,7 @@ class MergeRequestDiff < ActiveRecord::Base include Sortable include Importable - include EncodingHelper + include Gitlab::Git::EncodingHelper # Prevent store of diff if commits amount more then 500 COMMITS_SAFE_SIZE = 100 @@ -11,6 +11,9 @@ class MergeRequestDiff < ActiveRecord::Base belongs_to :merge_request + serialize :st_commits + serialize :st_diffs + state_machine :state, initial: :empty do state :collected state :overflow @@ -22,8 +25,7 @@ class MergeRequestDiff < ActiveRecord::Base state :overflow_diff_lines_limit end - serialize :st_commits - serialize :st_diffs + scope :viewable, -> { without_state(:empty) } # All diff information is collected from repository after object is created. # It allows you to override variables like head_commit_sha before getting diff. @@ -125,11 +127,7 @@ class MergeRequestDiff < ActiveRecord::Base end def commits_sha - if @commits - commits.map(&:sha) - else - st_commits.map { |commit| commit[:id] } - end + st_commits.map { |commit| commit[:id] } end def diff_refs @@ -174,6 +172,10 @@ class MergeRequestDiff < ActiveRecord::Base CompareService.new.execute(project, head_commit_sha, project, sha, straight: straight) end + def commits_count + st_commits.count + end + private # Old GitLab implementations may have generated diffs as ["--broken-diff"]. @@ -232,28 +234,28 @@ class MergeRequestDiff < ActiveRecord::Base # and save it as array of hashes in st_diffs db field def save_diffs new_attributes = {} - new_diffs = [] if commits.size.zero? new_attributes[:state] = :empty else diff_collection = compare.diffs(Commit.max_diff_options) - - if diff_collection.overflow? - # Set our state to 'overflow' to make the #empty? and #collected? - # methods (generated by StateMachine) return false. - new_attributes[:state] = :overflow - end - - new_attributes[:real_size] = diff_collection.real_size + new_attributes[:real_size] = compare.diffs.real_size if diff_collection.any? new_diffs = dump_diffs(diff_collection) new_attributes[:state] = :collected end + + new_attributes[:st_diffs] = new_diffs || [] + + # Set our state to 'overflow' to make the #empty? and #collected? + # methods (generated by StateMachine) return false. + # + # This attribution has to come at the end of the method so 'overflow' + # state does not get overridden by 'collected'. + new_attributes[:state] = :overflow if diff_collection.overflow? end - new_attributes[:st_diffs] = new_diffs update_columns_serialized(new_attributes) end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 23aecbfa3a6..7331000a9f2 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -29,6 +29,7 @@ class Milestone < ActiveRecord::Base validates :title, presence: true, uniqueness: { scope: :project_id } validates :project, presence: true + validate :start_date_should_be_less_than_due_date, if: Proc.new { |m| m.start_date.present? && m.due_date.present? } strip_attributes :title @@ -112,41 +113,24 @@ class Milestone < ActiveRecord::Base # # Examples: # - # Milestone.first.to_reference # => "%1" - # Milestone.first.to_reference(format: :name) # => "%\"goal\"" - # Milestone.first.to_reference(project) # => "gitlab-org/gitlab-ce%1" + # Milestone.first.to_reference # => "%1" + # Milestone.first.to_reference(format: :name) # => "%\"goal\"" + # Milestone.first.to_reference(cross_namespace_project) # => "gitlab-org/gitlab-ce%1" + # Milestone.first.to_reference(same_namespace_project) # => "gitlab-ce%1" # - def to_reference(from_project = nil, format: :iid) + def to_reference(from_project = nil, format: :iid, full: false) format_reference = milestone_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if cross_project_reference?(from_project) - project.to_reference + reference - else - reference - end + "#{project.to_reference(from_project, full: full)}#{reference}" end def reference_link_text(from_project = nil) self.title end - def expired? - if due_date - due_date.past? - else - false - end - end - - def expires_at - if due_date - if due_date.past? - "expired on #{due_date.to_s(:medium)}" - else - "expires on #{due_date.to_s(:medium)}" - end - end + def milestoneish_ids + id end def can_be_closed? @@ -212,4 +196,14 @@ class Milestone < ActiveRecord::Base def sanitize_title(value) CGI.unescape_html(Sanitize.clean(value.to_s)) end + + def start_date_should_be_less_than_due_date + if due_date <= start_date + errors.add(:start_date, "Can't be greater than due date") + end + end + + def issues_finder_params + { project_id: project_id } + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b67049f0f55..dd33975731f 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -4,29 +4,34 @@ class Namespace < ActiveRecord::Base include CacheMarkdownField include Sortable include Gitlab::ShellAdapter + include Routable cache_markdown_field :description, pipeline: :description has_many :projects, dependent: :destroy + has_many :project_statistics belongs_to :owner, class_name: "User" + belongs_to :parent, class_name: "Namespace" + has_many :children, class_name: "Namespace", foreign_key: :parent_id + validates :owner, presence: true, unless: ->(n) { n.type == "Group" } validates :name, - length: { within: 0..255 }, - namespace_name: true, presence: true, - uniqueness: true + uniqueness: { scope: :parent_id }, + length: { maximum: 255 }, + namespace_name: true - validates :description, length: { within: 0..255 } + validates :description, length: { maximum: 255 } validates :path, - length: { within: 1..255 }, - namespace: true, presence: true, - uniqueness: { case_sensitive: false } + length: { maximum: 255 }, + namespace: true delegate :name, to: :owner, allow_nil: true, prefix: true after_update :move_dir, if: :path_changed? + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } # Save the storage paths before the projects are destroyed to use them on after destroy before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths } @@ -34,6 +39,18 @@ class Namespace < ActiveRecord::Base scope :root, -> { where('type IS NULL') } + scope :with_statistics, -> do + joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id') + .group('namespaces.id') + .select( + 'namespaces.*', + 'COALESCE(SUM(ps.storage_size), 0) AS storage_size', + 'COALESCE(SUM(ps.repository_size), 0) AS repository_size', + 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', + 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', + ) + end + class << self def by_path(path) find_by('lower(path) = :value', value: path.downcase) @@ -85,7 +102,7 @@ class Namespace < ActiveRecord::Base end def to_param - path + full_path end def human_name @@ -94,7 +111,7 @@ class Namespace < ActiveRecord::Base def move_dir if any_project_has_container_registry_tags? - raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') end # Move the namespace directory in all storages paths used by member projects @@ -103,14 +120,18 @@ class Namespace < ActiveRecord::Base gitlab_shell.add_namespace(repository_storage_path, path_was) unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) + Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs - raise Exception.new('namespace directory cannot be moved') + raise Gitlab::UpdatePathError.new('namespace directory cannot be moved') end end Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) + remove_exports! + # If repositories moved successfully we need to # send update instructions to users. # However we cannot allow rollback since we moved namespace dir @@ -147,6 +168,27 @@ class Namespace < ActiveRecord::Base Gitlab.config.lfs.enabled end + def full_path + if parent + parent.full_path + '/' + path + else + path + end + end + + def full_name + @full_name ||= + if parent + parent.full_name + ' / ' + name + else + name + end + end + + def parents + @parents ||= parent ? parent.parents + [parent] : [] + end + private def repository_storage_paths @@ -174,5 +216,34 @@ class Namespace < ActiveRecord::Base GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) end end + + remove_exports! + end + + def refresh_access_of_projects_invited_groups + Group. + joins(project_group_links: :project). + where(projects: { namespace_id: id }). + find_each(&:refresh_members_authorized_projects) + end + + def full_path_changed? + path_changed? || parent_id_changed? + end + + def remove_exports! + Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) + end + + def export_path + File.join(Gitlab::ImportExport.storage_path, full_path_was) + end + + def full_path_was + if parent + parent.full_path + '/' + path_was + else + path_was + end end end diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 345041a6ad1..b524ca50ee8 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -161,8 +161,8 @@ module Network def is_overlap?(range, overlap_space) range.each do |i| if i != range.first && - i != range.last && - @commits[i].spaces.include?(overlap_space) + i != range.last && + @commits[i].spaces.include?(overlap_space) return true end diff --git a/app/models/note.rb b/app/models/note.rb index 2d644b03e4d..0c1b05dabf2 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -7,6 +7,7 @@ class Note < ActiveRecord::Base include Importable include FasterCacheKeys include CacheMarkdownField + include AfterCommitQueue cache_markdown_field :note, pipeline: :note @@ -18,6 +19,9 @@ class Note < ActiveRecord::Base # Banzai::ObjectRenderer attr_accessor :user_visible_reference_count + # Attribute used to store the attributes that have ben changed by slash commands. + attr_accessor :commands_changes + default_value_for :system, false attr_mentionable :note, pipeline: :note @@ -95,7 +99,7 @@ class Note < ActiveRecord::Base end def discussions - Discussion.for_notes(all) + Discussion.for_notes(fresh) end def grouped_diff_discussions @@ -103,23 +107,6 @@ class Note < ActiveRecord::Base Discussion.for_diff_notes(active_notes). map { |d| [d.line_code, d] }.to_h end - - # Searches for notes matching the given query. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String. - # as_user - Limit results to those viewable by a specific user - # - # Returns an ActiveRecord::Relation. - def search(query, as_user: nil) - table = arel_table - pattern = "%#{query}%" - - Note.joins('LEFT JOIN issues ON issues.id = noteable_id'). - where(table[:note].matches(pattern)). - merge(Issue.visible_to_user(as_user)) - end end def cross_reference? @@ -197,19 +184,6 @@ class Note < ActiveRecord::Base super(noteable_type.to_s.classify.constantize.base_class.to_s) end - # Reset notes events cache - # - # Since we do cache @event we need to reset cache in special cases: - # * when a note is updated - # * when a note is removed - # Events cache stored like events/23-20130109142513. - # The cache key includes updated_at timestamp. - # Thus it will automatically generate a new fragment - # when the event is updated because the key changes. - def reset_events_cache - Event.reset_event_cache_for(self) - end - def editable? !system? end diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 121b598b8f3..58f6214bea7 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -32,7 +32,13 @@ class NotificationSetting < ActiveRecord::Base :reopen_merge_request, :close_merge_request, :reassign_merge_request, - :merge_merge_request + :merge_merge_request, + :failed_pipeline, + :success_pipeline + ] + + EXCLUDED_WATCHER_EVENTS = [ + :success_pipeline ] store :events, accessors: EMAIL_EVENTS, coder: JSON diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index c4b095e0c04..10a34c42fd8 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -2,6 +2,8 @@ class PersonalAccessToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token + serialize :scopes, Array + belongs_to :user scope :active, -> { where(revoked: false).where("expires_at >= NOW() OR expires_at IS NULL") } diff --git a/app/models/project.rb b/app/models/project.rb index d5512dfaf9c..1630975b0d3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -12,7 +12,10 @@ class Project < ActiveRecord::Base include AfterCommitQueue include CaseSensitivity include TokenAuthenticatable + include ValidAttribute include ProjectFeaturesCompatibility + include SelectForProjectAuthorization + include Routable extend Gitlab::ConfigHelper @@ -23,22 +26,26 @@ class Project < ActiveRecord::Base cache_markdown_field :description, pipeline: :description - delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true + delegate :feature_available?, :builds_enabled?, :wiki_enabled?, + :merge_requests_enabled?, :issues_enabled?, to: :project_feature, + allow_nil: true default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :container_registry_enabled, gitlab_config_features.container_registry - default_value_for(:repository_storage) { current_application_settings.repository_storage } + default_value_for(:repository_storage) { current_application_settings.pick_repository_storage } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } default_value_for :issues_enabled, gitlab_config_features.issues default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests default_value_for :builds_enabled, gitlab_config_features.builds default_value_for :wiki_enabled, gitlab_config_features.wiki default_value_for :snippets_enabled, gitlab_config_features.snippets + default_value_for :only_allow_merge_if_all_discussions_are_resolved, false after_create :ensure_dir_exist after_create :create_project_feature, unless: :project_feature after_save :ensure_dir_exist, if: :namespace_id_changed? + after_save :update_project_statistics, if: :namespace_id_changed? # set last_activity_at to the same as created_at after_create :set_last_activity_at @@ -59,6 +66,8 @@ class Project < ActiveRecord::Base end end + after_validation :check_pending_delete + ActsAsTaggableOn.strict_case_match = true acts_as_taggable_on :tags @@ -76,7 +85,6 @@ class Project < ActiveRecord::Base has_many :boards, before_add: :validate_board_limit, dependent: :destroy # Project services - has_many :services has_one :campfire_service, dependent: :destroy has_one :drone_ci_service, dependent: :destroy has_one :emails_on_push_service, dependent: :destroy @@ -89,6 +97,9 @@ class Project < ActiveRecord::Base has_one :assembla_service, dependent: :destroy has_one :asana_service, dependent: :destroy has_one :gemnasium_service, dependent: :destroy + has_one :mattermost_slash_commands_service, dependent: :destroy + has_one :mattermost_service, dependent: :destroy + has_one :slack_slash_commands_service, dependent: :destroy has_one :slack_service, dependent: :destroy has_one :buildkite_service, dependent: :destroy has_one :bamboo_service, dependent: :destroy @@ -100,6 +111,7 @@ class Project < ActiveRecord::Base has_one :bugzilla_service, dependent: :destroy has_one :gitlab_issue_tracker_service, dependent: :destroy, inverse_of: :project has_one :external_wiki_service, dependent: :destroy + has_one :kubernetes_service, dependent: :destroy, inverse_of: :project has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" has_one :forked_from_project, through: :forked_project_link @@ -110,7 +122,7 @@ class Project < ActiveRecord::Base # Merge Requests for target project should be removed with it has_many :merge_requests, dependent: :destroy, foreign_key: 'target_project_id' # Merge requests from source project should be kept when source project was removed - has_many :fork_merge_requests, foreign_key: 'source_project_id', class_name: MergeRequest + has_many :fork_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' has_many :issues, dependent: :destroy has_many :labels, dependent: :destroy, class_name: 'ProjectLabel' has_many :services, dependent: :destroy @@ -121,6 +133,8 @@ class Project < ActiveRecord::Base has_many :hooks, dependent: :destroy, class_name: 'ProjectHook' has_many :protected_branches, dependent: :destroy + has_many :project_authorizations + has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :project_members has_many :users, through: :project_members @@ -141,6 +155,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy + has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id @@ -158,18 +173,20 @@ class Project < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, to: :team + delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team # Validations validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true validates :name, presence: true, - length: { within: 0..255 }, + length: { maximum: 255 }, format: { with: Gitlab::Regex.project_name_regex, message: Gitlab::Regex.project_name_regex_message } validates :path, presence: true, - length: { within: 0..255 }, + project_path: true, + length: { maximum: 255 }, format: { with: Gitlab::Regex.project_path_regex, message: Gitlab::Regex.project_path_regex_message } validates :namespace, presence: true @@ -207,8 +224,39 @@ class Project < ActiveRecord::Base scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } - scope :with_builds_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') } - scope :with_issues_enabled, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id').where('project_features.issues_access_level IS NULL or project_features.issues_access_level > 0') } + scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } + scope :with_statistics, -> { includes(:statistics) } + + # "enabled" here means "not disabled". It includes private features! + scope :with_feature_enabled, ->(feature) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => [nil, ProjectFeature::PRIVATE, ProjectFeature::ENABLED] }) + } + + # Picks a feature where the level is exactly that given. + scope :with_feature_access_level, ->(feature, level) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => level }) + } + + scope :with_builds_enabled, -> { with_feature_enabled(:builds) } + scope :with_issues_enabled, -> { with_feature_enabled(:issues) } + + # project features may be "disabled", "internal" or "enabled". If "internal", + # they are only available to team members. This scope returns projects where + # the feature is either enabled, or internal with permission for the user. + def self.with_feature_available_for_user(feature, user) + return with_feature_enabled(feature) if user.try(:admin?) + + unconditional = with_feature_access_level(feature, [nil, ProjectFeature::ENABLED]) + return unconditional if user.nil? + + conditional = with_feature_access_level(feature, ProjectFeature::PRIVATE) + authorized = user.authorized_projects.merge(conditional.reorder(nil)) + + union = Gitlab::SQL::Union.new([unconditional.select(:id), authorized.select(:id)]) + where(arel_table[:id].in(Arel::Nodes::SqlLiteral.new(union.to_sql))) + end scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } @@ -285,94 +333,15 @@ class Project < ActiveRecord::Base non_archived.where(table[:name].matches(pattern)) end - # Finds a single project for the given path. - # - # path - The full project path (including namespace path). - # - # Returns a Project, or nil if no project could be found. - def find_with_namespace(path) - namespace_path, project_path = path.split('/', 2) - - return unless namespace_path && project_path - - namespace_path = connection.quote(namespace_path) - project_path = connection.quote(project_path) - - # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so - # any literal matches come first, for this we have to use "BINARY". - # Without this there's still no guarantee in what order MySQL will return - # rows. - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - - order_sql = "(CASE WHEN #{binary} namespaces.path = #{namespace_path} " \ - "AND #{binary} projects.path = #{project_path} THEN 0 ELSE 1 END)" - - where_paths_in([path]).reorder(order_sql).take - end - - # Builds a relation to find multiple projects by their full paths. - # - # Each path must be in the following format: - # - # namespace_path/project_path - # - # For example: - # - # gitlab-org/gitlab-ce - # - # Usage: - # - # Project.where_paths_in(%w{gitlab-org/gitlab-ce gitlab-org/gitlab-ee}) - # - # This would return the projects with the full paths matching the values - # given. - # - # paths - An Array of full paths (namespace path + project path) for which - # to find the projects. - # - # Returns an ActiveRecord::Relation. - def where_paths_in(paths) - wheres = [] - cast_lower = Gitlab::Database.postgresql? - - paths.each do |path| - namespace_path, project_path = path.split('/', 2) - - next unless namespace_path && project_path - - namespace_path = connection.quote(namespace_path) - project_path = connection.quote(project_path) - - where = "(namespaces.path = #{namespace_path} - AND projects.path = #{project_path})" - - if cast_lower - where = "( - #{where} - OR ( - LOWER(namespaces.path) = LOWER(#{namespace_path}) - AND LOWER(projects.path) = LOWER(#{project_path}) - ) - )" - end - - wheres << where - end - - if wheres.empty? - none - else - joins(:namespace).where(wheres.join(' OR ')) - end - end - def visibility_levels Gitlab::VisibilityLevel.options end def sort(method) - if method == 'repository_size_desc' - reorder(repository_size: :desc, id: :desc) + if method == 'storage_size_desc' + # storage_size is a joined column so we need to + # pass a string to avoid AR adding the table name + reorder('project_statistics.storage_size DESC, projects.id DESC') else order_by(method) end @@ -380,7 +349,11 @@ class Project < ActiveRecord::Base def reference_pattern name_pattern = Gitlab::Regex::NAMESPACE_REGEX_STR - %r{(?<project>#{name_pattern}/#{name_pattern})} + + %r{ + ((?<namespace>#{name_pattern})\/)? + (?<project>#{name_pattern}) + }x end def trending @@ -397,6 +370,10 @@ class Project < ActiveRecord::Base def group_ids joins(:namespace).where(namespaces: { type: 'Group' }).select(:namespace_id) end + + # Add alias for Routable method for compatibility with old code. + # In future all calls `find_with_namespace` should be replaced with `find_by_full_path` + alias_method :find_with_namespace, :find_by_full_path end def lfs_enabled? @@ -564,6 +541,10 @@ class Project < ActiveRecord::Base import_type == 'gitlab_project' end + def gitea_import? + import_type == 'gitea' + end + def check_limit unless creator.can_create_project? or namespace.kind == 'group' projects_limit = creator.projects_limit @@ -611,26 +592,33 @@ class Project < ActiveRecord::Base end end - def to_reference(_from_project = nil) - path_with_namespace + def to_reference(from_project = nil, full: false) + if full || cross_namespace_reference?(from_project) + path_with_namespace + elsif cross_project_reference?(from_project) + path + end end - def web_url - Gitlab::Routing.url_helpers.namespace_project_url(self.namespace, self) + def to_human_reference(from_project = nil) + if cross_namespace_reference?(from_project) + name_with_namespace + elsif cross_project_reference?(from_project) + name + end end - def web_url_without_protocol - web_url.split('://')[1] + def web_url + Gitlab::Routing.url_helpers.namespace_project_url(self.namespace, self) end def new_issue_address(author) - # This feature is disabled for the time being. - return nil + return unless Gitlab::IncomingEmail.supports_issue_creation? && author - if Gitlab::IncomingEmail.enabled? && author # rubocop:disable Lint/UnreachableCode - Gitlab::IncomingEmail.reply_address( - "#{path_with_namespace}+#{author.authentication_token}") - end + author.ensure_incoming_email_token! + + Gitlab::IncomingEmail.reply_address( + "#{path_with_namespace}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -649,9 +637,9 @@ class Project < ActiveRecord::Base self.id end - def get_issue(issue_id) + def get_issue(issue_id, current_user) if default_issues_tracker? - issues.find_by(iid: issue_id) + IssuesFinder.new(current_user, project_id: id).find_by(iid: issue_id) else ExternalIssue.new(issue_id, self) end @@ -719,27 +707,32 @@ class Project < ActiveRecord::Base update_column(:has_external_wiki, services.external_wikis.any?) end - def build_missing_services + def find_or_initialize_services services_templates = Service.where(template: true) - Service.available_services_names.each do |service_name| + Service.available_services_names.map do |service_name| service = find_service(services, service_name) - # If service is available but missing in db - if service.nil? + if service + service + else # We should check if template for the service exists template = find_service(services_templates, service_name) if template.nil? - # If no template, we should create an instance. Ex `create_gitlab_ci_service` - public_send("create_#{service_name}_service") + # If no template, we should create an instance. Ex `build_gitlab_ci_service` + public_send("build_#{service_name}_service") else - Service.create_from_template(self.id, template) + Service.build_from_template(id, template) end end end end + def find_or_initialize_service(name) + find_or_initialize_services.find { |service| service.to_param == name } + end + def create_labels Label.templates.each do |label| params = label.attributes.except('id', 'template', 'created_at', 'updated_at') @@ -759,6 +752,14 @@ class Project < ActiveRecord::Base @ci_service ||= ci_services.reorder(nil).find_by(active: true) end + def deployment_services + services.where(category: :deployment) + end + + def deployment_service + @deployment_service ||= deployment_services.reorder(nil).find_by(active: true) + end + def jira_tracker? issues_tracker.to_param == 'jira' end @@ -820,13 +821,14 @@ class Project < ActiveRecord::Base end alias_method :human_name, :name_with_namespace - def path_with_namespace - if namespace - namespace.path + '/' + path + def full_path + if namespace && path + namespace.full_path + '/' + path else path end end + alias_method :path_with_namespace, :full_path def execute_hooks(data, hooks_scope = :push_hooks) hooks.send(hooks_scope).each do |hook| @@ -849,7 +851,7 @@ class Project < ActiveRecord::Base end def empty_repo? - !repository.exists? || !repository.has_visible_content? + repository.empty_repo? end def repo @@ -923,7 +925,7 @@ class Project < ActiveRecord::Base Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present" # we currently doesn't support renaming repository if it contains tags in container registry - raise Exception.new('Project cannot be renamed, because tags are present in its container registry') + raise StandardError.new('Project cannot be renamed, because tags are present in its container registry') end if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) @@ -933,7 +935,6 @@ class Project < ActiveRecord::Base begin gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") send_move_instructions(old_path_with_namespace) - reset_events_cache @old_path_with_namespace = old_path_with_namespace @@ -951,7 +952,7 @@ class Project < ActiveRecord::Base # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs - raise Exception.new('repository cannot be renamed') + raise StandardError.new('repository cannot be renamed') end Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" @@ -1000,22 +1001,6 @@ class Project < ActiveRecord::Base attrs end - # Reset events cache related to this project - # - # Since we do cache @event we need to reset cache in special cases: - # * when project was moved - # * when project was renamed - # * when the project avatar changes - # Events cache stored like events/23-20130109142513. - # The cache key includes updated_at timestamp. - # Thus it will automatically generate a new fragment - # when the event is updated because the key changes. - def reset_events_cache - Event.where(project_id: self.id). - order('id DESC').limit(100). - update_all(updated_at: Time.now) - end - def project_member(user) project_members.find_by(user_id: user) end @@ -1047,7 +1032,7 @@ class Project < ActiveRecord::Base "refs/heads/#{branch}", force: true) repository.copy_gitattributes(branch) - repository.expire_avatar_cache(branch) + repository.after_change_head reload_default_branch end @@ -1055,22 +1040,10 @@ class Project < ActiveRecord::Base forked? && project == forked_from_project end - def update_repository_size - update_attribute(:repository_size, repository.size) - end - - def update_commit_count - update_attribute(:commit_count, repository.commit_count) - end - def forks_count forks.count end - def find_label(name) - labels.find_by(name: name) - end - def origin_merge_requests merge_requests.where(source_project_id: self.id) end @@ -1257,18 +1230,10 @@ class Project < ActiveRecord::Base end end - # Checks if `user` is authorized for this project, with at least the - # `min_access_level` (if given). - # - # If you change the logic of this method, please also update `User#authorized_projects` - def authorized_for_user?(user, min_access_level = nil) - return false unless user - - return true if personal? && namespace_id == user.namespace_id + def deployment_variables + return [] unless deployment_service - authorized_for_user_by_group?(user, min_access_level) || - authorized_for_user_by_members?(user, min_access_level) || - authorized_for_user_by_shared_projects?(user, min_access_level) + deployment_service.predefined_variables end def append_or_update_attribute(name, value) @@ -1293,65 +1258,86 @@ class Project < ActiveRecord::Base Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } end - def environments_for(ref, commit, with_tags: false) - environment_ids = deployments.group(:environment_id). - select(:environment_id) + def environments_for(ref, commit: nil, with_tags: false) + deployments_query = with_tags ? 'ref = ? OR tag IS TRUE' : 'ref = ?' - environment_ids = - if with_tags - environment_ids.where('ref=? OR tag IS TRUE', ref) - else - environment_ids.where(ref: ref) - end + environment_ids = deployments + .where(deployments_query, ref.to_s) + .group(:environment_id) + .select(:environment_id) + + environments_found = environments.available + .where(id: environment_ids).to_a - environments.available.where(id: environment_ids).select do |environment| + return environments_found unless commit + + environments_found.select do |environment| environment.includes_commit?(commit) end end + def environments_recently_updated_on_branch(branch) + environments_for(branch).select do |environment| + environment.recently_updated_on_branch?(branch) + end + end + private + # Check if a reference is being done cross-project + # + # from_project - Refering Project object + def cross_project_reference?(from_project) + from_project && self != from_project + end + def pushes_since_gc_redis_key "projects/#{id}/pushes_since_gc" end + def cross_namespace_reference?(from_project) + from_project && namespace != from_project.namespace + end + def default_branch_protected? current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE end - def authorized_for_user_by_group?(user, min_access_level) - member = user.group_members.find_by(source_id: group) - - member && (!min_access_level || member.access_level >= min_access_level) + # Similar to the normal callbacks that hook into the life cycle of an + # Active Record object, you can also define callbacks that get triggered + # when you add an object to an association collection. If any of these + # callbacks throw an exception, the object will not be added to the + # collection. Before you add a new board to the boards collection if you + # already have 1, 2, or n it will fail, but it if you have 0 that is lower + # than the number of permitted boards per project it won't fail. + def validate_board_limit(board) + raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS end - def authorized_for_user_by_members?(user, min_access_level) - member = members.find_by(user_id: user) + def full_path_changed? + path_changed? || namespace_id_changed? + end - member && (!min_access_level || member.access_level >= min_access_level) + def update_project_statistics + stats = statistics || build_statistics + stats.update(namespace_id: namespace_id) end - def authorized_for_user_by_shared_projects?(user, min_access_level) - shared_projects = user.group_members.joins(group: :shared_projects). - where(project_group_links: { project_id: self }) + def check_pending_delete + return if valid_attribute?(:name) && valid_attribute?(:path) + return unless pending_delete_twin - if min_access_level - members_scope = { access_level: Gitlab::Access.values.select { |access| access >= min_access_level } } - shared_projects = shared_projects.where(members: members_scope) + %i[route route.path name path].each do |error| + errors.delete(error) end - shared_projects.any? + errors.add(:base, "The project is still being deleted. Please try again later.") end - # Similar to the normal callbacks that hook into the life cycle of an - # Active Record object, you can also define callbacks that get triggered - # when you add an object to an association collection. If any of these - # callbacks throw an exception, the object will not be added to the - # collection. Before you add a new board to the boards collection if you - # already have 1, 2, or n it will fail, but it if you have 0 that is lower - # than the number of permitted boards per project it won't fail. - def validate_board_limit(board) - raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS + def pending_delete_twin + return false unless path + + Project.unscoped.where(pending_delete: true).find_with_namespace(path_with_namespace) end end diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb new file mode 100644 index 00000000000..4c7f4f5a429 --- /dev/null +++ b/app/models/project_authorization.rb @@ -0,0 +1,21 @@ +class ProjectAuthorization < ActiveRecord::Base + belongs_to :user + belongs_to :project + + validates :project, presence: true + validates :access_level, inclusion: { in: Gitlab::Access.all_values }, presence: true + validates :user, uniqueness: { scope: [:project, :access_level] }, presence: true + + def self.insert_authorizations(rows, per_batch = 1000) + rows.each_slice(per_batch) do |slice| + tuples = slice.map do |tuple| + tuple.map { |value| connection.quote(value) } + end + + connection.execute <<-EOF.strip_heredoc + INSERT INTO project_authorizations (user_id, project_id, access_level) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + end + end +end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index b37ce1d3cf6..03194fc2141 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -20,6 +20,15 @@ class ProjectFeature < ActiveRecord::Base FEATURES = %i(issues merge_requests wiki snippets builds repository) + class << self + def access_level_attribute(feature) + feature = feature.model_name.plural.to_sym if feature.respond_to?(:model_name) + raise ArgumentError, "invalid project feature: #{feature}" unless FEATURES.include?(feature) + + "#{feature}_access_level".to_sym + end + end + # Default scopes force us to unscope here since a service may need to check # permissions for a project in pending_delete # http://stackoverflow.com/questions/1540645/how-to-disable-default-scope-for-a-belongs-to @@ -35,29 +44,26 @@ class ProjectFeature < ActiveRecord::Base default_value_for :repository_access_level, value: ENABLED, allows_nil: false def feature_available?(feature, user) - raise ArgumentError, 'invalid project feature' unless FEATURES.include?(feature) - - get_permission(user, public_send("#{feature}_access_level")) + access_level = public_send(ProjectFeature.access_level_attribute(feature)) + get_permission(user, access_level) end def builds_enabled? - return true unless builds_access_level - builds_access_level > DISABLED end def wiki_enabled? - return true unless wiki_access_level - wiki_access_level > DISABLED end def merge_requests_enabled? - return true unless merge_requests_access_level - merge_requests_access_level > DISABLED end + def issues_enabled? + issues_access_level > DISABLED + end + private # Validates builds and merge requests access level diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index db46def11eb..6149c35cc61 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -16,6 +16,9 @@ class ProjectGroupLink < ActiveRecord::Base validates :group_access, inclusion: { in: Gitlab::Access.values }, presence: true validate :different_group + after_create :refresh_group_members_authorized_projects + after_destroy :refresh_group_members_authorized_projects + def self.access_options Gitlab::Access.options end @@ -35,4 +38,8 @@ class ProjectGroupLink < ActiveRecord::Base errors.add(:base, "Project cannot be shared with the project it is in.") end end + + def refresh_group_members_authorized_projects + group.refresh_members_authorized_projects + end end diff --git a/app/models/project_label.rb b/app/models/project_label.rb index 82f47f0e8fd..313815e5869 100644 --- a/app/models/project_label.rb +++ b/app/models/project_label.rb @@ -16,8 +16,8 @@ class ProjectLabel < Label 'project_id' end - def to_reference(target_project = nil, format: :id) - super(project, target_project, format: format) + def to_reference(target_project = nil, format: :id, full: false) + super(project, target_project: target_project, format: format, full: full) end private diff --git a/app/models/project_services/asana_service.rb b/app/models/project_services/asana_service.rb index 7c23b766763..3728f5642e4 100644 --- a/app/models/project_services/asana_service.rb +++ b/app/models/project_services/asana_service.rb @@ -25,7 +25,7 @@ You can create a Personal Access Token here: http://app.asana.com/-/account_api' end - def to_param + def self.to_param 'asana' end @@ -44,7 +44,7 @@ http://app.asana.com/-/account_api' ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/assembla_service.rb b/app/models/project_services/assembla_service.rb index d839221d315..aeeff8917bf 100644 --- a/app/models/project_services/assembla_service.rb +++ b/app/models/project_services/assembla_service.rb @@ -12,7 +12,7 @@ class AssemblaService < Service 'Project Management Software (Source Commits Endpoint)' end - def to_param + def self.to_param 'assembla' end @@ -23,7 +23,7 @@ class AssemblaService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index b5c76e4d4fe..400020ee04a 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -1,4 +1,6 @@ class BambooService < CiService + include ReactiveService + prop_accessor :bamboo_url, :build_key, :username, :password validates :bamboo_url, presence: true, url: true, if: :activated? @@ -38,7 +40,7 @@ class BambooService < CiService 'You must set up automatic revision labeling and a repository trigger in Bamboo.' end - def to_param + def self.to_param 'bamboo' end @@ -54,35 +56,46 @@ class BambooService < CiService ] end - def supported_events - %w(push) + def build_page(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:build_page] } end - def build_info(sha) - @response = get_path("rest/api/latest/result?label=#{sha}") + def commit_status(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } end - def build_page(sha, ref) - build_info(sha) if @response.nil? || !@response.code + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + get_path("updateAndBuild.action?buildKey=#{build_key}") + end + + def calculate_reactive_cache(sha, ref) + response = get_path("rest/api/latest/result?label=#{sha}") + + { build_page: read_build_page(response), commit_status: read_commit_status(response) } + end - if @response.code != 200 || @response['results']['results']['size'] == '0' + private + + def read_build_page(response) + if response.code != 200 || response['results']['results']['size'] == '0' # If actual build link can't be determined, send user to build summary page. URI.join("#{bamboo_url}/", "browse/#{build_key}").to_s else # If actual build link is available, go to build result page. - result_key = @response['results']['results']['result']['planResultKey']['key'] + result_key = response['results']['results']['result']['planResultKey']['key'] URI.join("#{bamboo_url}/", "browse/#{result_key}").to_s end end - def commit_status(sha, ref) - build_info(sha) if @response.nil? || !@response.code - return :error unless @response.code == 200 || @response.code == 404 + def read_commit_status(response) + return :error unless response.code == 200 || response.code == 404 - status = if @response.code == 404 || @response['results']['results']['size'] == '0' + status = if response.code == 404 || response['results']['results']['size'] == '0' 'Pending' else - @response['results']['results']['result']['buildState'] + response['results']['results']['result']['buildState'] end if status.include?('Success') @@ -96,14 +109,6 @@ class BambooService < CiService end end - def execute(data) - return unless supported_events.include?(data[:object_kind]) - - get_path("updateAndBuild.action?buildKey=#{build_key}") - end - - private - def build_url(path) URI.join("#{bamboo_url}/", path).to_s end diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 338e685339a..046e2809f45 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -19,7 +19,7 @@ class BugzillaService < IssueTrackerService end end - def to_param + def self.to_param 'bugzilla' end end diff --git a/app/models/project_services/buildkite_service.rb b/app/models/project_services/buildkite_service.rb index 86a06321e21..0956c4a4ede 100644 --- a/app/models/project_services/buildkite_service.rb +++ b/app/models/project_services/buildkite_service.rb @@ -1,9 +1,12 @@ require "addressable/uri" class BuildkiteService < CiService + include ReactiveService + ENDPOINT = "https://buildkite.com" - prop_accessor :project_url, :token, :enable_ssl_verification + prop_accessor :project_url, :token + boolean_accessor :enable_ssl_verification validates :project_url, presence: true, url: true, if: :activated? validates :token, presence: true, if: :activated? @@ -21,10 +24,6 @@ class BuildkiteService < CiService hook.save end - def supported_events - %w(push) - end - def execute(data) return unless supported_events.include?(data[:object_kind]) @@ -32,13 +31,7 @@ class BuildkiteService < CiService end def commit_status(sha, ref) - response = HTTParty.get(commit_status_path(sha), verify: false) - - if response.code == 200 && response['status'] - response['status'] - else - :error - end + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } end def commit_status_path(sha) @@ -57,7 +50,7 @@ class BuildkiteService < CiService 'Continuous integration and deployments' end - def to_param + def self.to_param 'buildkite' end @@ -77,6 +70,19 @@ class BuildkiteService < CiService ] end + def calculate_reactive_cache(sha, ref) + response = HTTParty.get(commit_status_path(sha), verify: false) + + status = + if response.code == 200 && response['status'] + response['status'] + else + :error + end + + { commit_status: status } + end + private def webhook_token diff --git a/app/models/project_services/builds_email_service.rb b/app/models/project_services/builds_email_service.rb index 201b94b065b..ebd21e37189 100644 --- a/app/models/project_services/builds_email_service.rb +++ b/app/models/project_services/builds_email_service.rb @@ -19,11 +19,11 @@ class BuildsEmailService < Service 'Email the builds status to a list of recipients.' end - def to_param + def self.to_param 'builds_email' end - def supported_events + def self.supported_events %w(build) end diff --git a/app/models/project_services/campfire_service.rb b/app/models/project_services/campfire_service.rb index 5af93860d09..0de59af5652 100644 --- a/app/models/project_services/campfire_service.rb +++ b/app/models/project_services/campfire_service.rb @@ -12,7 +12,7 @@ class CampfireService < Service 'Simple web-based real-time group chat' end - def to_param + def self.to_param 'campfire' end @@ -24,7 +24,7 @@ class CampfireService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/slack_service/base_message.rb b/app/models/project_services/chat_message/base_message.rb index f1182824687..a03605d01fb 100644 --- a/app/models/project_services/slack_service/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -1,6 +1,6 @@ require 'slack-notifier' -class SlackService +module ChatMessage class BaseMessage def initialize(params) raise NotImplementedError diff --git a/app/models/project_services/slack_service/build_message.rb b/app/models/project_services/chat_message/build_message.rb index 0fca4267bad..53e35cb21bf 100644 --- a/app/models/project_services/slack_service/build_message.rb +++ b/app/models/project_services/chat_message/build_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class BuildMessage < BaseMessage attr_reader :sha attr_reader :ref_type diff --git a/app/models/project_services/slack_service/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index cd87a79d0c6..14fd64e5332 100644 --- a/app/models/project_services/slack_service/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class IssueMessage < BaseMessage attr_reader :user_name attr_reader :title diff --git a/app/models/project_services/slack_service/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index b7615c96068..ab5e8b24167 100644 --- a/app/models/project_services/slack_service/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class MergeMessage < BaseMessage attr_reader :user_name attr_reader :project_name diff --git a/app/models/project_services/slack_service/note_message.rb b/app/models/project_services/chat_message/note_message.rb index 9e84e90f38c..ca1d7207034 100644 --- a/app/models/project_services/slack_service/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class NoteMessage < BaseMessage attr_reader :message attr_reader :user_name @@ -46,25 +46,25 @@ class SlackService commit_sha = commit[:id] commit_sha = Commit.truncate_sha(commit_sha) commented_on_message( - "[commit #{commit_sha}](#{@note_url})", + "commit #{commit_sha}", format_title(commit[:message])) end def create_issue_note(issue) commented_on_message( - "[issue ##{issue[:iid]}](#{@note_url})", + "issue ##{issue[:iid]}", format_title(issue[:title])) end def create_merge_note(merge_request) commented_on_message( - "[merge request !#{merge_request[:iid]}](#{@note_url})", + "merge request !#{merge_request[:iid]}", format_title(merge_request[:title])) end def create_snippet_note(snippet) commented_on_message( - "[snippet ##{snippet[:id]}](#{@note_url})", + "snippet ##{snippet[:id]}", format_title(snippet[:title])) end @@ -76,8 +76,8 @@ class SlackService "[#{@project_name}](#{@project_url})" end - def commented_on_message(target_link, title) - @message = "#{@user_name} commented on #{target_link} in #{project_link}: *#{title}*" + def commented_on_message(target, title) + @message = "#{@user_name} [commented on #{target}](#{@note_url}) in #{project_link}: *#{title}*" end end end diff --git a/app/models/project_services/slack_service/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index f06b3562965..210027565a8 100644 --- a/app/models/project_services/slack_service/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -1,11 +1,10 @@ -class SlackService +module ChatMessage class PipelineMessage < BaseMessage - attr_reader :sha, :ref_type, :ref, :status, :project_name, :project_url, + attr_reader :ref_type, :ref, :status, :project_name, :project_url, :user_name, :duration, :pipeline_id def initialize(data) pipeline_attributes = data[:object_attributes] - @sha = pipeline_attributes[:sha] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] @@ -14,7 +13,7 @@ class SlackService @project_name = data[:project][:path_with_namespace] @project_url = data[:project][:web_url] - @user_name = data[:commit] && data[:commit][:author_name] + @user_name = (data[:user] && data[:user][:name]) || 'API' end def pretext @@ -73,7 +72,7 @@ class SlackService end def pipeline_link - "[#{Commit.truncate_sha(sha)}](#{pipeline_url})" + "[##{pipeline_id}](#{pipeline_url})" end end end diff --git a/app/models/project_services/slack_service/push_message.rb b/app/models/project_services/chat_message/push_message.rb index b26f3e9ddce..2d73b71ec37 100644 --- a/app/models/project_services/slack_service/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class PushMessage < BaseMessage attr_reader :after attr_reader :before diff --git a/app/models/project_services/slack_service/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index 160ca3ac115..134083e4504 100644 --- a/app/models/project_services/slack_service/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -1,4 +1,4 @@ -class SlackService +module ChatMessage class WikiPageMessage < BaseMessage attr_reader :user_name attr_reader :title diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb new file mode 100644 index 00000000000..8468934425f --- /dev/null +++ b/app/models/project_services/chat_notification_service.rb @@ -0,0 +1,149 @@ +# Base class for Chat notifications services +# This class is not meant to be used directly, but only to inherit from. +class ChatNotificationService < Service + include ChatMessage + + default_value_for :category, 'chat' + + prop_accessor :webhook, :username, :channel + boolean_accessor :notify_only_broken_builds, :notify_only_broken_pipelines + + validates :webhook, presence: true, url: true, if: :activated? + + def initialize_properties + # Custom serialized properties initialization + self.supported_events.each { |event| self.class.prop_accessor(event_channel_name(event)) } + + if properties.nil? + self.properties = {} + self.notify_only_broken_builds = true + self.notify_only_broken_pipelines = true + end + end + + def can_test? + valid? + end + + def self.supported_events + %w[push issue confidential_issue merge_request note tag_push + build pipeline wiki_page] + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + return unless webhook.present? + + object_kind = data[:object_kind] + + data = data.merge( + project_url: project_url, + project_name: project_name + ) + + # WebHook events often have an 'update' event that follows a 'open' or + # 'close' action. Ignore update events for now to prevent duplicate + # messages from arriving. + + message = get_message(object_kind, data) + + return false unless message + + channel_name = get_channel_field(object_kind).presence || channel + + opts = {} + opts[:channel] = channel_name if channel_name + opts[:username] = username if username + + notifier = Slack::Notifier.new(webhook, opts) + notifier.ping(message.pretext, attachments: message.attachments, fallback: message.fallback) + + true + end + + def event_channel_names + supported_events.map { |event| event_channel_name(event) } + end + + def event_field(event) + fields.find { |field| field[:name] == event_channel_name(event) } + end + + def global_fields + fields.reject { |field| field[:name].end_with?('channel') } + end + + def default_channel_placeholder + raise NotImplementedError + end + + private + + def get_message(object_kind, data) + case object_kind + when "push", "tag_push" + ChatMessage::PushMessage.new(data) + when "issue" + ChatMessage::IssueMessage.new(data) unless is_update?(data) + when "merge_request" + ChatMessage::MergeMessage.new(data) unless is_update?(data) + when "note" + ChatMessage::NoteMessage.new(data) + when "build" + ChatMessage::BuildMessage.new(data) if should_build_be_notified?(data) + when "pipeline" + ChatMessage::PipelineMessage.new(data) if should_pipeline_be_notified?(data) + when "wiki_page" + ChatMessage::WikiPageMessage.new(data) + end + end + + def get_channel_field(event) + field_name = event_channel_name(event) + self.public_send(field_name) + end + + def build_event_channels + supported_events.reduce([]) do |channels, event| + channels << { type: 'text', name: event_channel_name(event), placeholder: default_channel_placeholder } + end + end + + def event_channel_name(event) + "#{event}_channel" + end + + def project_name + project.name_with_namespace.gsub(/\s/, '') + end + + def project_url + project.web_url + end + + def is_update?(data) + data[:object_attributes][:action] == 'update' + end + + def should_build_be_notified?(data) + case data[:commit][:status] + when 'success' + !notify_only_broken_builds? + when 'failed' + true + else + false + end + end + + def should_pipeline_be_notified?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end +end diff --git a/app/models/project_services/chat_slash_commands_service.rb b/app/models/project_services/chat_slash_commands_service.rb new file mode 100644 index 00000000000..2bcff541cc0 --- /dev/null +++ b/app/models/project_services/chat_slash_commands_service.rb @@ -0,0 +1,56 @@ +# Base class for Chat services +# This class is not meant to be used directly, but only to inherrit from. +class ChatSlashCommandsService < Service + default_value_for :category, 'chat' + + prop_accessor :token + + has_many :chat_names, foreign_key: :service_id, dependent: :destroy + + def valid_token?(token) + self.respond_to?(:token) && + self.token.present? && + ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) + end + + def self.supported_events + %w() + end + + def can_test? + false + end + + def fields + [ + { type: 'text', name: 'token', placeholder: '' } + ] + end + + def trigger(params) + return unless valid_token?(params[:token]) + + user = find_chat_user(params) + unless user + url = authorize_chat_name_url(params) + return presenter.authorize_chat_name(url) + end + + Gitlab::ChatCommands::Command.new(project, user, + params).execute + end + + private + + def find_chat_user(params) + ChatNames::FindUserService.new(self, params).execute + end + + def authorize_chat_name_url(params) + ChatNames::AuthorizeUserService.new(self, params).execute + end + + def presenter + Gitlab::ChatCommands::Presenter.new + end +end diff --git a/app/models/project_services/ci_service.rb b/app/models/project_services/ci_service.rb index 596c00705ad..82979c8bd34 100644 --- a/app/models/project_services/ci_service.rb +++ b/app/models/project_services/ci_service.rb @@ -8,19 +8,11 @@ class CiService < Service self.respond_to?(:token) && self.token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) end - def supported_events + def self.supported_events %w(push) end - def merge_request_page(iid, sha, ref) - commit_page(sha, ref) - end - - def commit_page(sha, ref) - build_page(sha, ref) - end - - # Return complete url to merge_request page + # Return complete url to build page # # Ex. # http://jenkins.example.com:8888/job/test1/scm/bySHA1/12d65c @@ -35,23 +27,6 @@ class CiService < Service # # # Ex. - # @service.merge_request_status(9, '13be4ac', 'dev') - # # => 'success' - # - # @service.merge_request_status(10, '2abe4ac', 'dev) - # # => 'running' - # - # - def merge_request_status(iid, sha, ref) - commit_status(sha, ref) - end - - # Return string with build status or :error symbol - # - # Allowed states: 'success', 'failed', 'running', 'pending', 'skipped' - # - # - # Ex. # @service.commit_status('13be4ac', 'master') # # => 'success' # diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index b2f426dc2ac..dea915a4d05 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -23,7 +23,7 @@ class CustomIssueTrackerService < IssueTrackerService end end - def to_param + def self.to_param 'custom_issue_tracker' end diff --git a/app/models/project_services/deployment_service.rb b/app/models/project_services/deployment_service.rb new file mode 100644 index 00000000000..91a55514a9a --- /dev/null +++ b/app/models/project_services/deployment_service.rb @@ -0,0 +1,33 @@ +# Base class for deployment services +# +# These services integrate with a deployment solution like Kubernetes/OpenShift, +# Mesosphere, etc, to provide additional features to environments. +class DeploymentService < Service + default_value_for :category, 'deployment' + + def self.supported_events + %w() + end + + def predefined_variables + [] + end + + # Environments may have a number of terminals. Should return an array of + # hashes describing them, e.g.: + # + # [{ + # :selectors => {"a" => "b", "foo" => "bar"}, + # :url => "wss://external.example.com/exec", + # :headers => {"Authorization" => "Token xxx"}, + # :subprotocols => ["foo"], + # :ca_pem => "----BEGIN CERTIFICATE...", # optional + # :created_at => Time.now.utc + # }] + # + # Selectors should be a set of values that uniquely identify a particular + # terminal + def terminals(environment) + raise NotImplementedError + end +end diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index 5e4dd101c53..0a217d8caba 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -1,5 +1,8 @@ class DroneCiService < CiService - prop_accessor :drone_url, :token, :enable_ssl_verification + include ReactiveService + + prop_accessor :drone_url, :token + boolean_accessor :enable_ssl_verification validates :drone_url, presence: true, url: true, if: :activated? validates :token, presence: true, if: :activated? @@ -29,18 +32,10 @@ class DroneCiService < CiService true end - def supported_events + def self.supported_events %w(push merge_request tag_push) end - def merge_request_status_path(iid, sha = nil, ref = nil) - url = [drone_url, - "gitlab/#{project.namespace.path}/#{project.path}/pulls/#{iid}", - "?access_token=#{token}"] - - URI.join(*url).to_s - end - def commit_status_path(sha, ref) url = [drone_url, "gitlab/#{project.namespace.path}/#{project.path}/commits/#{sha}", @@ -49,54 +44,34 @@ class DroneCiService < CiService URI.join(*url).to_s end - def merge_request_status(iid, sha, ref) - response = HTTParty.get(merge_request_status_path(iid), verify: enable_ssl_verification) - - if response.code == 200 and response['status'] - case response['status'] - when 'killed' - :canceled - when 'failure', 'error' - # Because drone return error if some test env failed - :failed - else - response["status"] - end - else - :error - end - rescue Errno::ECONNREFUSED - :error + def commit_status(sha, ref) + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } end - def commit_status(sha, ref) + def calculate_reactive_cache(sha, ref) response = HTTParty.get(commit_status_path(sha, ref), verify: enable_ssl_verification) - if response.code == 200 and response['status'] - case response['status'] - when 'killed' - :canceled - when 'failure', 'error' - # Because drone return error if some test env failed - :failed + status = + if response.code == 200 and response['status'] + case response['status'] + when 'killed' + :canceled + when 'failure', 'error' + # Because drone return error if some test env failed + :failed + else + response["status"] + end else - response["status"] + :error end - else - :error - end - rescue Errno::ECONNREFUSED - :error - end - def merge_request_page(iid, sha, ref) - url = [drone_url, - "gitlab/#{project.namespace.path}/#{project.path}/redirect/pulls/#{iid}"] - - URI.join(*url).to_s + { commit_status: status } + rescue Errno::ECONNREFUSED + { commit_status: :error } end - def commit_page(sha, ref) + def build_page(sha, ref) url = [drone_url, "gitlab/#{project.namespace.path}/#{project.path}/redirect/commits/#{sha}", "?branch=#{URI::encode(ref.to_s)}"] @@ -104,14 +79,6 @@ class DroneCiService < CiService URI.join(*url).to_s end - def commit_coverage(sha, ref) - nil - end - - def build_page(sha, ref) - commit_page(sha, ref) - end - def title 'Drone CI' end @@ -120,7 +87,7 @@ class DroneCiService < CiService 'Drone is a Continuous Integration platform built on Docker, written in Go' end - def to_param + def self.to_param 'drone_ci' end diff --git a/app/models/project_services/emails_on_push_service.rb b/app/models/project_services/emails_on_push_service.rb index e0083c43adb..f4f913ee0b6 100644 --- a/app/models/project_services/emails_on_push_service.rb +++ b/app/models/project_services/emails_on_push_service.rb @@ -1,6 +1,6 @@ class EmailsOnPushService < Service - prop_accessor :send_from_committer_email - prop_accessor :disable_diffs + boolean_accessor :send_from_committer_email + boolean_accessor :disable_diffs prop_accessor :recipients validates :recipients, presence: true, if: :activated? @@ -12,11 +12,11 @@ class EmailsOnPushService < Service 'Email the commits and diff of each push to a list of recipients.' end - def to_param + def self.to_param 'emails_on_push' end - def supported_events + def self.supported_events %w(push tag_push) end @@ -24,20 +24,20 @@ class EmailsOnPushService < Service return unless supported_events.include?(push_data[:object_kind]) EmailsOnPushWorker.perform_async( - project_id, - recipients, - push_data, - send_from_committer_email: send_from_committer_email?, - disable_diffs: disable_diffs? + project_id, + recipients, + push_data, + send_from_committer_email: send_from_committer_email?, + disable_diffs: disable_diffs? ) end def send_from_committer_email? - self.send_from_committer_email == "1" + Gitlab::Utils.to_boolean(self.send_from_committer_email) end def disable_diffs? - self.disable_diffs == "1" + Gitlab::Utils.to_boolean(self.disable_diffs) end def fields diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb index d7b6e505191..bdf6fa6a586 100644 --- a/app/models/project_services/external_wiki_service.rb +++ b/app/models/project_services/external_wiki_service.rb @@ -13,7 +13,7 @@ class ExternalWikiService < Service 'Replaces the link to the internal wiki with a link to an external wiki.' end - def to_param + def self.to_param 'external_wiki' end @@ -29,4 +29,8 @@ class ExternalWikiService < Service nil end end + + def self.supported_events + %w() + end end diff --git a/app/models/project_services/flowdock_service.rb b/app/models/project_services/flowdock_service.rb index dd00275187f..10a13c3fbdc 100644 --- a/app/models/project_services/flowdock_service.rb +++ b/app/models/project_services/flowdock_service.rb @@ -12,7 +12,7 @@ class FlowdockService < Service 'Flowdock is a collaboration web app for technical teams.' end - def to_param + def self.to_param 'flowdock' end @@ -22,7 +22,7 @@ class FlowdockService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 598aca5e06d..f271e1f1739 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -12,7 +12,7 @@ class GemnasiumService < Service 'Gemnasium monitors your project dependencies and alerts you about updates and security vulnerabilities.' end - def to_param + def self.to_param 'gemnasium' end @@ -23,7 +23,7 @@ class GemnasiumService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index 6bd8d4ec568..ad4eb9536e1 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -7,7 +7,7 @@ class GitlabIssueTrackerService < IssueTrackerService default_value_for :default, true - def to_param + def self.to_param 'gitlab' end diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 660a8ae3421..72da219df28 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -8,8 +8,8 @@ class HipchatService < Service ul ol li dl dt dd ] - prop_accessor :token, :room, :server, :notify, :color, :api_version - boolean_accessor :notify_only_broken_builds + prop_accessor :token, :room, :server, :color, :api_version + boolean_accessor :notify_only_broken_builds, :notify validates :token, presence: true, if: :activated? def initialize_properties @@ -27,7 +27,7 @@ class HipchatService < Service 'Private group chat and IM' end - def to_param + def self.to_param 'hipchat' end @@ -45,7 +45,7 @@ class HipchatService < Service ] end - def supported_events + def self.supported_events %w(push issue confidential_issue merge_request note tag_push build) end @@ -75,7 +75,7 @@ class HipchatService < Service end def message_options(data = nil) - { notify: notify.present? && notify == '1', color: message_color(data) } + { notify: notify.present? && Gitlab::Utils.to_boolean(notify), color: message_color(data) } end def create_message(data) diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb index ce7d1c5d5b1..5d93064f9b3 100644 --- a/app/models/project_services/irker_service.rb +++ b/app/models/project_services/irker_service.rb @@ -2,7 +2,8 @@ require 'uri' class IrkerService < Service prop_accessor :server_host, :server_port, :default_irc_uri - prop_accessor :colorize_messages, :recipients, :channels + prop_accessor :recipients, :channels + boolean_accessor :colorize_messages validates :recipients, presence: true, if: :activated? before_validation :get_channels @@ -16,11 +17,11 @@ class IrkerService < Service 'gateway.' end - def to_param + def self.to_param 'irker' end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index 207bb816ad1..9e65fdbf9d6 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -57,7 +57,7 @@ class IssueTrackerService < Service end end - def supported_events + def self.supported_events %w(push) end @@ -85,8 +85,8 @@ class IssueTrackerService < Service def enabled_in_gitlab_config Gitlab.config.issues_tracker && - Gitlab.config.issues_tracker.values.any? && - issues_tracker + Gitlab.config.issues_tracker.values.any? && + issues_tracker end def issues_tracker diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 5bcf199d468..2ac76e97de0 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -1,24 +1,3 @@ -# == Schema Information -# -# Table name: services -# -# id :integer not null, primary key -# type :string(255) -# title :string(255) -# project_id :integer -# created_at :datetime -# updated_at :datetime -# active :boolean default(FALSE), not null -# properties :text -# template :boolean default(FALSE) -# push_events :boolean default(TRUE) -# issues_events :boolean default(TRUE) -# merge_requests_events :boolean default(TRUE) -# tag_push_events :boolean default(TRUE) -# note_events :boolean default(TRUE), not null -# build_events :boolean default(FALSE), not null -# - class JiraService < IssueTrackerService include Gitlab::Routing.url_helpers @@ -30,6 +9,13 @@ class JiraService < IssueTrackerService before_update :reset_password + # This is confusing, but JiraService does not really support these events. + # The values here are required to display correct options in the service + # configuration screen. + def self.supported_events + %w(commit merge_request) + end + # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 def reference_pattern @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} @@ -70,13 +56,13 @@ class JiraService < IssueTrackerService end def jira_project - @jira_project ||= client.Project.find(project_key) + @jira_project ||= jira_request { client.Project.find(project_key) } end def help - 'See the ' \ - '[integration doc](http://doc.gitlab.com/ce/integration/external-issue-tracker.html) '\ - 'for details.' + 'You need to configure JIRA before enabling this service. For more details + read the + [JIRA service documentation](https://docs.gitlab.com/ce/project_services/jira.html).' end def title @@ -95,7 +81,7 @@ class JiraService < IssueTrackerService end end - def to_param + def self.to_param 'jira' end @@ -122,27 +108,43 @@ class JiraService < IssueTrackerService "#{url}/secure/CreateIssue.jspa" end - def execute(push, issue = nil) - if issue.nil? - # No specific issue, that means - # we just want to test settings - test_settings - else - close_issue(push, issue) - end + def execute(push) + # This method is a no-op, because currently JiraService does not + # support any events. + end + + def close_issue(entity, external_issue) + issue = jira_request { client.Issue.find(external_issue.iid) } + + return if issue.nil? || issue.resolution.present? || !jira_issue_transition_id.present? + + commit_id = if entity.is_a?(Commit) + entity.id + elsif entity.is_a?(MergeRequest) + entity.diff_head_sha + end + + commit_url = build_entity_url(:commit, commit_id) + + # Depending on the JIRA project's workflow, a comment during transition + # may or may not be allowed. Refresh the issue after transition and check + # if it is closed, so we don't have one comment for every commit. + issue = jira_request { client.Issue.find(issue.key) } if transition_issue(issue) + add_issue_solved_comment(issue, commit_id, commit_url) if issue.resolution end def create_cross_reference_note(mentioned, noteable, author) - issue_key = mentioned.id - project = self.project - noteable_name = noteable.class.name.underscore.downcase - noteable_id = if noteable.is_a?(Commit) - noteable.id - else - noteable.iid - end + unless can_cross_reference?(noteable) + return "Events for #{noteable.model_name.plural.humanize(capitalize: false)} are disabled." + end - entity_url = build_entity_url(noteable_name.to_sym, noteable_id) + jira_issue = jira_request { client.Issue.find(mentioned.id) } + + return unless jira_issue.present? + + noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id + noteable_type = noteable_name(noteable) + entity_url = build_entity_url(noteable_type, noteable_id) data = { user: { @@ -150,107 +152,160 @@ class JiraService < IssueTrackerService url: resource_url(user_path(author)), }, project: { - name: project.path_with_namespace, - url: resource_url(namespace_project_path(project.namespace, project)) + name: self.project.path_with_namespace, + url: resource_url(namespace_project_path(project.namespace, self.project)) }, entity: { - name: noteable_name.humanize.downcase, + name: noteable_type.humanize.downcase, url: entity_url, title: noteable.title } } - add_comment(data, issue_key) + add_comment(data, jira_issue) + end + + # reason why service cannot be tested + def disabled_title + "Please fill in Password and Username." + end + + def test(_) + result = test_settings + { success: result.present?, result: result } + end + + def can_test? + username.present? && password.present? + end + + # JIRA does not need test data. + # We are requesting the project that belongs to the project key. + def test_data(user = nil, project = nil) + nil end def test_settings return unless url.present? # Test settings by getting the project - jira_project - - rescue Errno::ECONNREFUSED, JIRA::HTTPError => e - Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{url}." - false + jira_request { jira_project.present? } end private - def close_issue(entity, issue) - commit_id = if entity.is_a?(Commit) - entity.id - elsif entity.is_a?(MergeRequest) - entity.diff_head_sha - end - - commit_url = build_entity_url(:commit, commit_id) - - # Depending on the JIRA project's workflow, a comment during transition - # may or may not be allowed. Split the operation in to two calls so the - # comment always works. - transition_issue(issue) - add_issue_solved_comment(issue, commit_id, commit_url) + def can_cross_reference?(noteable) + case noteable + when Commit then commit_events + when MergeRequest then merge_requests_events + else true + end end def transition_issue(issue) - issue = client.Issue.find(issue.iid) issue.transitions.build.save(transition: { id: jira_issue_transition_id }) end def add_issue_solved_comment(issue, commit_id, commit_url) - comment = "Issue solved with [#{commit_id}|#{commit_url}]." - send_message(issue.iid, comment) + link_title = "GitLab: Solved by commit #{commit_id}." + comment = "Issue solved with [#{commit_id}|#{commit_url}]." + link_props = build_remote_link_props(url: commit_url, title: link_title, resolved: true) + send_message(issue, comment, link_props) end - def add_comment(data, issue_key) - user_name = data[:user][:name] - user_url = data[:user][:url] - entity_name = data[:entity][:name] - entity_url = data[:entity][:url] + def add_comment(data, issue) + user_name = data[:user][:name] + user_url = data[:user][:url] + entity_name = data[:entity][:name] + entity_url = data[:entity][:url] entity_title = data[:entity][:title] project_name = data[:project][:name] - message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'" + message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title.chomp}'" + link_title = "GitLab: Mentioned on #{entity_name} - #{entity_title}" + link_props = build_remote_link_props(url: entity_url, title: link_title) - unless comment_exists?(issue_key, message) - send_message(issue_key, message) + unless comment_exists?(issue, message) + send_message(issue, message, link_props) end end - def comment_exists?(issue_key, message) - comments = client.Issue.find(issue_key).comments - comments.map { |comment| comment.body.include?(message) }.any? + def comment_exists?(issue, message) + comments = jira_request { issue.comments } + + comments.present? && comments.any? { |comment| comment.body.include?(message) } end - def send_message(issue_key, message) + def send_message(issue, message, remote_link_props) return unless url.present? - issue = client.Issue.find(issue_key) + jira_request do + if issue.comments.build.save!(body: message) + remote_link = issue.remotelink.build + remote_link.save!(remote_link_props) + result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}." + end - if issue.comments.build.save!(body: message) - result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}." + Rails.logger.info(result_message) + result_message end + end - Rails.logger.info(result_message) - result_message - rescue URI::InvalidURIError, Errno::ECONNREFUSED, JIRA::HTTPError => e - Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}" + # Build remote link on JIRA properties + # Icons here must be available on WEB so JIRA can read the URL + # We are using a open word graphics icon which have LGPL license + def build_remote_link_props(url:, title:, resolved: false) + status = { + resolved: resolved + } + + if resolved + status[:icon] = { + title: 'Closed', + url16x16: 'http://www.openwebgraphics.com/resources/data/1768/16x16_apply.png' + } + end + + { + GlobalID: 'GitLab', + object: { + url: url, + title: title, + status: status, + icon: { title: 'GitLab', url16x16: 'https://gitlab.com/favicon.ico' } + } + } end def resource_url(resource) - "#{Settings.gitlab['url'].chomp("/")}#{resource}" - end - - def build_entity_url(entity_name, entity_id) - resource_url( - polymorphic_url( - [ - self.project.namespace.becomes(Namespace), - self.project, - entity_name - ], - id: entity_id, - routing_type: :path - ) + "#{Settings.gitlab.base_url.chomp("/")}#{resource}" + end + + def build_entity_url(noteable_type, entity_id) + polymorphic_url( + [ + self.project.namespace.becomes(Namespace), + self.project, + noteable_type.to_sym + ], + id: entity_id, + host: Settings.gitlab.base_url ) end + + def noteable_name(noteable) + name = noteable.model_name.singular + + # ProjectSnippet inherits from Snippet class so it causes + # routing error building the URL. + name == "project_snippet" ? "snippet" : name + end + + # Handle errors when doing JIRA API calls + def jira_request + yield + + rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, URI::InvalidURIError, JIRA::HTTPError => e + Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}" + nil + end end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb new file mode 100644 index 00000000000..fa3cedc4354 --- /dev/null +++ b/app/models/project_services/kubernetes_service.rb @@ -0,0 +1,173 @@ +class KubernetesService < DeploymentService + include Gitlab::Kubernetes + include ReactiveCaching + + self.reactive_cache_key = ->(service) { [ service.class.model_name.singular, service.project_id ] } + + # Namespace defaults to the project path, but can be overridden in case that + # is an invalid or inappropriate name + prop_accessor :namespace + + # Access to kubernetes is directly through the API + prop_accessor :api_url + + # Bearer authentication + # TODO: user/password auth, client certificates + prop_accessor :token + + # Provide a custom CA bundle for self-signed deployments + prop_accessor :ca_pem + + with_options presence: true, if: :activated? do + validates :api_url, url: true + validates :token + + validates :namespace, + format: { + with: Gitlab::Regex.kubernetes_namespace_regex, + message: Gitlab::Regex.kubernetes_namespace_regex_message, + }, + length: 1..63 + end + + after_save :clear_reactive_cache! + + def initialize_properties + if properties.nil? + self.properties = {} + self.namespace = project.path if project.present? + end + end + + def title + 'Kubernetes' + end + + def description + 'Kubernetes / Openshift integration' + end + + def help + 'To enable terminal access to Kubernetes environments, label your ' \ + 'deployments with `app=$CI_ENVIRONMENT_SLUG`' + end + + def self.to_param + 'kubernetes' + end + + def fields + [ + { type: 'text', + name: 'namespace', + title: 'Kubernetes namespace', + placeholder: 'Kubernetes namespace', + }, + { type: 'text', + name: 'api_url', + title: 'API URL', + placeholder: 'Kubernetes API URL, like https://kube.example.com/', + }, + { type: 'text', + name: 'token', + title: 'Service token', + placeholder: 'Service token', + }, + { type: 'textarea', + name: 'ca_pem', + title: 'Custom CA bundle', + placeholder: 'Certificate Authority bundle (PEM format)', + }, + ] + end + + # Check we can connect to the Kubernetes API + def test(*args) + kubeclient = build_kubeclient! + + kubeclient.discover + { success: kubeclient.discovered, result: "Checked API discovery endpoint" } + rescue => err + { success: false, result: err } + end + + def predefined_variables + variables = [ + { key: 'KUBE_URL', value: api_url, public: true }, + { key: 'KUBE_TOKEN', value: token, public: false }, + { key: 'KUBE_NAMESPACE', value: namespace, public: true } + ] + variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true } if ca_pem.present? + variables + end + + # Constructs a list of terminals from the reactive cache + # + # Returns nil if the cache is empty, in which case you should try again a + # short time later + def terminals(environment) + with_reactive_cache do |data| + pods = data.fetch(:pods, nil) + filter_pods(pods, app: environment.slug). + flat_map { |pod| terminals_for_pod(api_url, namespace, pod) }. + map { |terminal| add_terminal_auth(terminal, token, ca_pem) } + end + end + + # Caches all pods in the namespace so other calls don't need to block on + # network access. + def calculate_reactive_cache + return unless active? && project && !project.pending_delete? + + kubeclient = build_kubeclient! + + # Store as hashes, rather than as third-party types + pods = begin + kubeclient.get_pods(namespace: namespace).as_json + rescue KubeException => err + raise err unless err.error_code == 404 + [] + end + + # We may want to cache extra things in the future + { pods: pods } + end + + private + + def build_kubeclient!(api_path: 'api', api_version: 'v1') + raise "Incomplete settings" unless api_url && namespace && token + + ::Kubeclient::Client.new( + join_api_url(api_path), + api_version, + auth_options: kubeclient_auth_options, + ssl_options: kubeclient_ssl_options, + http_proxy_uri: ENV['http_proxy'] + ) + end + + def kubeclient_ssl_options + opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } + + if ca_pem.present? + opts[:cert_store] = OpenSSL::X509::Store.new + opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) + end + + opts + end + + def kubeclient_auth_options + { bearer_token: token } + end + + def join_api_url(*parts) + url = URI.parse(api_url) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [ prefix, *parts ].join("/") + + url.to_s + end +end diff --git a/app/models/project_services/mattermost_service.rb b/app/models/project_services/mattermost_service.rb new file mode 100644 index 00000000000..4ebc5318da1 --- /dev/null +++ b/app/models/project_services/mattermost_service.rb @@ -0,0 +1,41 @@ +class MattermostService < ChatNotificationService + def title + 'Mattermost notifications' + end + + def description + 'Receive event notifications in Mattermost' + end + + def self.to_param + 'mattermost' + end + + def help + 'This service sends notifications about projects events to Mattermost channels.<br /> + To set up this service: + <ol> + <li><a href="https://docs.mattermost.com/developer/webhooks-incoming.html#enabling-incoming-webhooks">Enable incoming webhooks</a> in your Mattermost installation. </li> + <li><a href="https://docs.mattermost.com/developer/webhooks-incoming.html#creating-integrations-using-incoming-webhooks">Add an incoming webhook</a> in your Mattermost team. The default channel can be overridden for each event. </li> + <li>Paste the webhook <strong>URL</strong> into the field bellow. </li> + <li>Select events below to enable notifications. The channel and username are optional. </li> + </ol>' + end + + def fields + default_fields + build_event_channels + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: 'http://mattermost_host/hooks/...' }, + { type: 'text', name: 'username', placeholder: 'username' }, + { type: 'checkbox', name: 'notify_only_broken_builds' }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + ] + end + + def default_channel_placeholder + "town-square" + end +end diff --git a/app/models/project_services/mattermost_slash_commands_service.rb b/app/models/project_services/mattermost_slash_commands_service.rb new file mode 100644 index 00000000000..50a011db74e --- /dev/null +++ b/app/models/project_services/mattermost_slash_commands_service.rb @@ -0,0 +1,51 @@ +class MattermostSlashCommandsService < ChatSlashCommandsService + include TriggersHelper + + prop_accessor :token + + def can_test? + false + end + + def title + 'Mattermost Command' + end + + def description + "Perform common operations on GitLab in Mattermost" + end + + def self.to_param + 'mattermost_slash_commands' + end + + def configure(user, params) + token = Mattermost::Command.new(user). + create(command(params)) + + update(active: true, token: token) if token + rescue Mattermost::Error => e + [false, e.message] + end + + def list_teams(user) + Mattermost::Team.new(user).all + rescue Mattermost::Error => e + [[], e.message] + end + + private + + def command(params) + pretty_project_name = project.name_with_namespace + + params.merge( + auto_complete: true, + auto_complete_desc: "Perform common operations on: #{pretty_project_name}", + auto_complete_hint: '[help]', + description: "Perform common operations on: #{pretty_project_name}", + display_name: "GitLab / #{pretty_project_name}", + method: 'P', + username: 'GitLab') + end +end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb index ec3c1bc85ee..ac617f409d9 100644 --- a/app/models/project_services/pipelines_email_service.rb +++ b/app/models/project_services/pipelines_email_service.rb @@ -1,10 +1,7 @@ class PipelinesEmailService < Service prop_accessor :recipients - boolean_accessor :add_pusher boolean_accessor :notify_only_broken_pipelines - validates :recipients, - presence: true, - if: ->(s) { s.activated? && !s.add_pusher? } + validates :recipients, presence: true, if: :activated? def initialize_properties self.properties ||= { notify_only_broken_pipelines: true } @@ -18,11 +15,11 @@ class PipelinesEmailService < Service 'Email the pipelines status to a list of recipients.' end - def to_param + def self.to_param 'pipelines_email' end - def supported_events + def self.supported_events %w[pipeline] end @@ -34,8 +31,8 @@ class PipelinesEmailService < Service return unless all_recipients.any? - pipeline = Ci::Pipeline.find(data[:object_attributes][:id]) - Ci::SendPipelineNotificationService.new(pipeline).execute(all_recipients) + pipeline_id = data[:object_attributes][:id] + PipelineNotificationWorker.new.perform(pipeline_id, all_recipients) end def can_test? @@ -58,9 +55,6 @@ class PipelinesEmailService < Service name: 'recipients', placeholder: 'Emails separated by comma' }, { type: 'checkbox', - name: 'add_pusher', - label: 'Add pusher to recipients list' }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, ] end @@ -85,12 +79,6 @@ class PipelinesEmailService < Service end def retrieve_recipients(data) - all_recipients = recipients.to_s.split(',').reject(&:blank?) - - if add_pusher? && data[:user].try(:[], :email) - all_recipients << data[:user][:email] - end - - all_recipients + recipients.to_s.split(',').reject(&:blank?) end end diff --git a/app/models/project_services/pivotaltracker_service.rb b/app/models/project_services/pivotaltracker_service.rb index 5301f9fa0ff..9cc642591f4 100644 --- a/app/models/project_services/pivotaltracker_service.rb +++ b/app/models/project_services/pivotaltracker_service.rb @@ -14,7 +14,7 @@ class PivotaltrackerService < Service 'Project Management Software (Source Commits Endpoint)' end - def to_param + def self.to_param 'pivotaltracker' end @@ -34,7 +34,7 @@ class PivotaltrackerService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/pushover_service.rb b/app/models/project_services/pushover_service.rb index 3dd878e4c7d..a963d27a376 100644 --- a/app/models/project_services/pushover_service.rb +++ b/app/models/project_services/pushover_service.rb @@ -13,7 +13,7 @@ class PushoverService < Service 'Pushover makes it easy to get real-time notifications on your Android device, iPhone, iPad, and Desktop.' end - def to_param + def self.to_param 'pushover' end @@ -61,7 +61,7 @@ class PushoverService < Service ] end - def supported_events + def self.supported_events %w(push) end diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index f9da273cf08..6acf611eba5 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -19,7 +19,7 @@ class RedmineService < IssueTrackerService end end - def to_param + def self.to_param 'redmine' end end diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb index e1b937817f4..f77d2d7c60b 100644 --- a/app/models/project_services/slack_service.rb +++ b/app/models/project_services/slack_service.rb @@ -1,176 +1,40 @@ -class SlackService < Service - prop_accessor :webhook, :username, :channel - boolean_accessor :notify_only_broken_builds, :notify_only_broken_pipelines - validates :webhook, presence: true, url: true, if: :activated? - - def initialize_properties - # Custom serialized properties initialization - self.supported_events.each { |event| self.class.prop_accessor(event_channel_name(event)) } - - if properties.nil? - self.properties = {} - self.notify_only_broken_builds = true - self.notify_only_broken_pipelines = true - end - end - +class SlackService < ChatNotificationService def title - 'Slack' + 'Slack notifications' end def description - 'A team communication tool for the 21st century' + 'Receive event notifications in Slack' end - def to_param + def self.to_param 'slack' end def help - 'This service sends notifications to your Slack channel.<br/> - To setup this Service you need to create a new <b>"Incoming webhook"</b> in your Slack integration panel, - and enter the Webhook URL below.' + 'This service sends notifications about projects events to Slack channels.<br /> + To setup this service: + <ol> + <li><a href="https://slack.com/apps/A0F7XDUAZ-incoming-webhooks">Add an incoming webhook</a> in your Slack team. The default channel can be overridden for each event. </li> + <li>Paste the <strong>Webhook URL</strong> into the field below. </li> + <li>Select events below to enable notifications. The channel and username are optional. </li> + </ol>' end def fields - default_fields = - [ - { type: 'text', name: 'webhook', placeholder: 'https://hooks.slack.com/services/...' }, - { type: 'text', name: 'username', placeholder: 'username' }, - { type: 'text', name: 'channel', placeholder: "#general" }, - { type: 'checkbox', name: 'notify_only_broken_builds' }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - ] - default_fields + build_event_channels end - def supported_events - %w[push issue confidential_issue merge_request note tag_push - build pipeline wiki_page] - end - - def execute(data) - return unless supported_events.include?(data[:object_kind]) - return unless webhook.present? - - object_kind = data[:object_kind] - - data = data.merge( - project_url: project_url, - project_name: project_name - ) - - # WebHook events often have an 'update' event that follows a 'open' or - # 'close' action. Ignore update events for now to prevent duplicate - # messages from arriving. - - message = get_message(object_kind, data) - - if message - opt = {} - - event_channel = get_channel_field(object_kind) || channel - - opt[:channel] = event_channel if event_channel - opt[:username] = username if username - - notifier = Slack::Notifier.new(webhook, opt) - notifier.ping(message.pretext, attachments: message.attachments, fallback: message.fallback) - - true - else - false - end - end - - def event_channel_names - supported_events.map { |event| event_channel_name(event) } - end - - def event_field(event) - fields.find { |field| field[:name] == event_channel_name(event) } + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: 'https://hooks.slack.com/services/...' }, + { type: 'text', name: 'username', placeholder: 'username' }, + { type: 'checkbox', name: 'notify_only_broken_builds' }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + ] end - def global_fields - fields.reject { |field| field[:name].end_with?('channel') } - end - - private - - def get_message(object_kind, data) - case object_kind - when "push", "tag_push" - PushMessage.new(data) - when "issue" - IssueMessage.new(data) unless is_update?(data) - when "merge_request" - MergeMessage.new(data) unless is_update?(data) - when "note" - NoteMessage.new(data) - when "build" - BuildMessage.new(data) if should_build_be_notified?(data) - when "pipeline" - PipelineMessage.new(data) if should_pipeline_be_notified?(data) - when "wiki_page" - WikiPageMessage.new(data) - end - end - - def get_channel_field(event) - field_name = event_channel_name(event) - self.public_send(field_name) - end - - def build_event_channels - supported_events.reduce([]) do |channels, event| - channels << { type: 'text', name: event_channel_name(event), placeholder: "#general" } - end - end - - def event_channel_name(event) - "#{event}_channel" - end - - def project_name - project.name_with_namespace.gsub(/\s/, '') - end - - def project_url - project.web_url - end - - def is_update?(data) - data[:object_attributes][:action] == 'update' - end - - def should_build_be_notified?(data) - case data[:commit][:status] - when 'success' - !notify_only_broken_builds? - when 'failed' - true - else - false - end - end - - def should_pipeline_be_notified?(data) - case data[:object_attributes][:status] - when 'success' - !notify_only_broken_pipelines? - when 'failed' - true - else - false - end + def default_channel_placeholder + "#general" end end - -require "slack_service/issue_message" -require "slack_service/push_message" -require "slack_service/merge_message" -require "slack_service/note_message" -require "slack_service/build_message" -require "slack_service/pipeline_message" -require "slack_service/wiki_page_message" diff --git a/app/models/project_services/slack_slash_commands_service.rb b/app/models/project_services/slack_slash_commands_service.rb new file mode 100644 index 00000000000..c34991e4262 --- /dev/null +++ b/app/models/project_services/slack_slash_commands_service.rb @@ -0,0 +1,28 @@ +class SlackSlashCommandsService < ChatSlashCommandsService + include TriggersHelper + + def title + 'Slack Command' + end + + def description + "Perform common operations on GitLab in Slack" + end + + def self.to_param + 'slack_slash_commands' + end + + def trigger(params) + # Format messages to be Slack-compatible + super.tap do |result| + result[:text] = format(result[:text]) if result.is_a?(Hash) + end + end + + private + + def format(text) + Slack::Notifier::LinkFormatter.format(text) if text + end +end diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index a4a967c9bc9..cbaffb8ce48 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -1,4 +1,6 @@ class TeamcityService < CiService + include ReactiveService + prop_accessor :teamcity_url, :build_type, :username, :password validates :teamcity_url, presence: true, url: true, if: :activated? @@ -41,14 +43,10 @@ class TeamcityService < CiService 'requests build, that setting is in the vsc root advanced settings.' end - def to_param + def self.to_param 'teamcity' end - def supported_events - %w(push) - end - def fields [ { type: 'text', name: 'teamcity_url', @@ -61,43 +59,18 @@ class TeamcityService < CiService ] end - def build_info(sha) - @response = get_path("httpAuth/app/rest/builds/branch:unspecified:any,number:#{sha}") - end - def build_page(sha, ref) - build_info(sha) if @response.nil? || !@response.code - - if @response.code != 200 - # If actual build link can't be determined, - # send user to build summary page. - build_url("viewLog.html?buildTypeId=#{build_type}") - else - # If actual build link is available, go to build result page. - built_id = @response['build']['id'] - build_url("viewLog.html?buildId=#{built_id}&buildTypeId=#{build_type}") - end + with_reactive_cache(sha, ref) {|cached| cached[:build_page] } end def commit_status(sha, ref) - build_info(sha) if @response.nil? || !@response.code - return :error unless @response.code == 200 || @response.code == 404 + with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + end - status = if @response.code == 404 - 'Pending' - else - @response['build']['status'] - end + def calculate_reactive_cache(sha, ref) + response = get_path("httpAuth/app/rest/builds/branch:unspecified:any,number:#{sha}") - if status.include?('SUCCESS') - 'success' - elsif status.include?('FAILURE') - 'failed' - elsif status.include?('Pending') - 'pending' - else - :error - end + { build_page: read_build_page(response), commit_status: read_commit_status(response) } end def execute(data) @@ -122,6 +95,40 @@ class TeamcityService < CiService private + def read_build_page(response) + if response.code != 200 + # If actual build link can't be determined, + # send user to build summary page. + build_url("viewLog.html?buildTypeId=#{build_type}") + else + # If actual build link is available, go to build result page. + built_id = response['build']['id'] + build_url("viewLog.html?buildId=#{built_id}&buildTypeId=#{build_type}") + end + end + + def read_commit_status(response) + return :error unless response.code == 200 || response.code == 404 + + status = if response.code == 404 + 'Pending' + else + response['build']['status'] + end + + return :error unless status.present? + + if status.include?('SUCCESS') + 'success' + elsif status.include?('FAILURE') + 'failed' + elsif status.include?('Pending') + 'pending' + else + :error + end + end + def build_url(path) URI.join("#{teamcity_url}/", path).to_s end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb new file mode 100644 index 00000000000..06abd406523 --- /dev/null +++ b/app/models/project_statistics.rb @@ -0,0 +1,44 @@ +class ProjectStatistics < ActiveRecord::Base + belongs_to :project + belongs_to :namespace + + before_save :update_storage_size + + STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size] + STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS + + def total_repository_size + repository_size + lfs_objects_size + end + + def refresh!(only: nil) + STATISTICS_COLUMNS.each do |column, generator| + if only.blank? || only.include?(column) + public_send("update_#{column}") + end + end + + save! + end + + def update_commit_count + self.commit_count = project.repository.commit_count + end + + # Repository#size needs to be converted from MB to Byte. + def update_repository_size + self.repository_size = project.repository.size * 1.megabyte + end + + def update_lfs_objects_size + self.lfs_objects_size = project.lfs_objects.sum(:size) + end + + def update_build_artifacts_size + self.build_artifacts_size = project.builds.sum(:artifacts_size) + end + + def update_storage_size + self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute)) + end +end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index a6e911df9bd..8a53e974b6f 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -21,6 +21,22 @@ class ProjectTeam end end + def add_guest(user, current_user: nil) + self << [user, :guest, current_user] + end + + def add_reporter(user, current_user: nil) + self << [user, :reporter, current_user] + end + + def add_developer(user, current_user: nil) + self << [user, :developer, current_user] + end + + def add_master(user, current_user: nil) + self << [user, :master, current_user] + end + def find_member(user_id) member = project.members.find_by(user_id: user_id) @@ -64,19 +80,19 @@ class ProjectTeam alias_method :users, :members def guests - @guests ||= fetch_members(:guests) + @guests ||= fetch_members(Gitlab::Access::GUEST) end def reporters - @reporters ||= fetch_members(:reporters) + @reporters ||= fetch_members(Gitlab::Access::REPORTER) end def developers - @developers ||= fetch_members(:developers) + @developers ||= fetch_members(Gitlab::Access::DEVELOPER) end def masters - @masters ||= fetch_members(:masters) + @masters ||= fetch_members(Gitlab::Access::MASTER) end def import(source_project, current_user = nil) @@ -125,8 +141,12 @@ class ProjectTeam max_member_access(user.id) == Gitlab::Access::MASTER end - def member?(user, min_member_access = Gitlab::Access::GUEST) - max_member_access(user.id) >= min_member_access + # Checks if `user` is authorized for this project, with at least the + # `min_access_level` (if given). + def member?(user, min_access_level = Gitlab::Access::GUEST) + return false unless user + + user.authorized_project?(project, min_access_level) end def human_max_access(user_id) @@ -149,112 +169,29 @@ class ProjectTeam # Lookup only the IDs we need user_ids = user_ids - access.keys + users_access = project.project_authorizations. + where(user: user_ids). + group(:user_id). + maximum(:access_level) - if user_ids.present? - user_ids.each { |id| access[id] = Gitlab::Access::NO_ACCESS } - - member_access = project.members.access_for_user_ids(user_ids) - merge_max!(access, member_access) - - if group - group_access = group.members.access_for_user_ids(user_ids) - merge_max!(access, group_access) - end - - # Each group produces a list of maximum access level per user. We take the - # max of the values produced by each group. - if project_shared_with_group? - project.project_group_links.each do |group_link| - invited_access = max_invited_level_for_users(group_link, user_ids) - merge_max!(access, invited_access) - end - end - end - + access.merge!(users_access) access end def max_member_access(user_id) - max_member_access_for_user_ids([user_id])[user_id] + max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS end private - # For a given group, return the maximum access level for the user. This is the min of - # the invited access level of the group and the access level of the user within the group. - # For example, if the group has been given DEVELOPER access but the member has MASTER access, - # the user should receive only DEVELOPER access. - def max_invited_level_for_users(group_link, user_ids) - invited_group = group_link.group - capped_access_level = group_link.group_access - access = invited_group.group_members.access_for_user_ids(user_ids) - - # If the user is not in the list, assume he/she does not have access - missing_users = user_ids - access.keys - missing_users.each { |id| access[id] = Gitlab::Access::NO_ACCESS } - - # Cap the maximum access by the invited level access - access.each { |key, value| access[key] = [value, capped_access_level].min } - end - def fetch_members(level = nil) - project_members = project.members - group_members = group ? group.members : [] - - if level - project_members = project_members.public_send(level) - group_members = group_members.public_send(level) if group - end - - user_ids = project_members.pluck(:user_id) - - invited_members = fetch_invited_members(level) - user_ids.push(*invited_members.map(&:user_id)) if invited_members.any? + members = project.authorized_users + members = members.where(project_authorizations: { access_level: level }) if level - user_ids.push(*group_members.pluck(:user_id)) if group - - User.where(id: user_ids) + members end def group project.group end - - def merge_max!(first_hash, second_hash) - first_hash.merge!(second_hash) { |_key, old, new| old > new ? old : new } - end - - def project_shared_with_group? - project.invited_groups.any? && project.allowed_to_share_with_group? - end - - def fetch_invited_members(level = nil) - invited_members = [] - - return invited_members unless project_shared_with_group? - - project.project_group_links.includes(group: [:group_members]).each do |link| - invited_group_members = link.group.members - - if level - numeric_level = GroupMember.access_level_roles[level.to_s.singularize.titleize] - - # If we're asked for a level that's higher than the group's access, - # there's nothing left to do - next if numeric_level > link.group_access - - # Make sure we include everyone _above_ the requested level as well - invited_group_members = - if numeric_level == link.group_access - invited_group_members.where("access_level >= ?", link.group_access) - else - invited_group_members.public_send(level) - end - end - - invited_members << invited_group_members - end - - invited_members.flatten.compact - end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 46f70da2452..9db96347322 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -127,7 +127,7 @@ class ProjectWiki end def search_files(query) - repository.search_files(query, default_branch) + repository.search_files_by_content(query, default_branch) end def repository diff --git a/app/models/protected_branch/merge_access_level.rb b/app/models/protected_branch/merge_access_level.rb index 806b3ccd275..771e3376613 100644 --- a/app/models/protected_branch/merge_access_level.rb +++ b/app/models/protected_branch/merge_access_level.rb @@ -1,9 +1,6 @@ class ProtectedBranch::MergeAccessLevel < ActiveRecord::Base include ProtectedBranchAccess - belongs_to :protected_branch - delegate :project, to: :protected_branch - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, Gitlab::Access::DEVELOPER] } @@ -13,10 +10,4 @@ class ProtectedBranch::MergeAccessLevel < ActiveRecord::Base Gitlab::Access::DEVELOPER => "Developers + Masters" }.with_indifferent_access end - - def check_access(user) - return true if user.is_admin? - - project.team.max_member_access(user.id) >= access_level - end end diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index 92e9c51d883..14610cb42b7 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -1,9 +1,6 @@ class ProtectedBranch::PushAccessLevel < ActiveRecord::Base include ProtectedBranchAccess - belongs_to :protected_branch - delegate :project, to: :protected_branch - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, Gitlab::Access::DEVELOPER, Gitlab::Access::NO_ACCESS] } @@ -18,8 +15,7 @@ class ProtectedBranch::PushAccessLevel < ActiveRecord::Base def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS - return true if user.is_admin? - project.team.max_member_access(user.id) >= access_level + super end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 30be7262438..43dba86e5ed 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1,28 +1,56 @@ require 'securerandom' class Repository - class CommitError < StandardError; end - - # Files to use as a project avatar in case no avatar was uploaded via the web - # UI. - AVATAR_FILES = %w{logo.png logo.jpg logo.gif} - include Gitlab::ShellAdapter attr_accessor :path_with_namespace, :project - def self.storages - Gitlab.config.repositories.storages - end + class CommitError < StandardError; end - def self.remove_storage_from_path(repo_path) - storages.find do |_, storage_path| - if repo_path.start_with?(storage_path) - return repo_path.sub(storage_path, '') - end + # Methods that cache data from the Git repository. + # + # Each entry in this Array should have a corresponding method with the exact + # same name. The cache key used by those methods must also match method's + # name. + # + # For example, for entry `:readme` there's a method called `readme` which + # stores its data in the `readme` cache key. + CACHED_METHODS = %i(size commit_count readme version contribution_guide + changelog license_blob license_key gitignore koding_yml + gitlab_ci_yml branch_names tag_names branch_count + tag_count avatar exists? empty? root_ref) + + # Certain method caches should be refreshed when certain types of files are + # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to + # the corresponding methods to call for refreshing caches. + METHOD_CACHES_FOR_FILE_TYPES = { + readme: :readme, + changelog: :changelog, + license: %i(license_blob license_key), + contributing: :contribution_guide, + version: :version, + gitignore: :gitignore, + koding: :koding_yml, + gitlab_ci: :gitlab_ci_yml, + avatar: :avatar + } + + # Wraps around the given method and caches its output in Redis and an instance + # variable. + # + # This only works for methods that do not take any arguments. + def self.cache_method(name, fallback: nil) + original = :"_uncached_#{name}" + + alias_method(original, name) + + define_method(name) do + cache_method_output(name, fallback: fallback) { __send__(original) } end + end - repo_path + def self.storages + Gitlab.config.repositories.storages end def initialize(path_with_namespace, project) @@ -47,24 +75,6 @@ class Repository ) end - def exists? - return @exists unless @exists.nil? - - @exists = cache.fetch(:exists?) do - begin - raw_repository && raw_repository.rugged ? true : false - rescue Gitlab::Git::Repository::NoRepository - false - end - end - end - - def empty? - return @empty unless @empty.nil? - - @empty = cache.fetch(:empty?) { raw_repository.empty? } - end - # # Git repository can contains some hidden refs like: # /refs/notes/* @@ -75,24 +85,22 @@ class Repository # This method return true if repository contains some content visible in project page. # def has_visible_content? - return @has_visible_content unless @has_visible_content.nil? - - @has_visible_content = cache.fetch(:has_visible_content?) do - branch_count > 0 - end + branch_count > 0 end def commit(ref = 'HEAD') return nil unless exists? + commit = if ref.is_a?(Gitlab::Git::Commit) ref else Gitlab::Git::Commit.find(raw_repository, ref) end + commit = ::Commit.new(commit, @project) if commit commit - rescue Rugged::OdbError + rescue Rugged::OdbError, Rugged::TreeError nil end @@ -184,8 +192,9 @@ class Repository options = { message: message, tagger: user_to_committer(user) } if message - GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do - rugged.tags.create(tag_name, target, options) + GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do |service| + raw_tag = rugged.tags.create(tag_name, target, options) + service.newrev = raw_tag.target_id end find_tag(tag_name) @@ -222,16 +231,14 @@ class Repository branch_names + tag_names end - def branch_names - @branch_names ||= cache.fetch(:branch_names) { branches.map(&:name) } - end - def branch_exists?(branch_name) branch_names.include?(branch_name) end def ref_exists?(ref) rugged.references.exist?(ref) + rescue Rugged::ReferenceError + false end def update_ref!(name, newrev, oldrev) @@ -239,7 +246,7 @@ class Repository # offer 'compare and swap' ref updates. Without compare-and-swap we can # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %w[git update-ref --stdin -z] + command = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z) _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin| stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00") end @@ -270,39 +277,7 @@ class Repository end def kept_around?(sha) - begin - ref_exists?(keep_around_ref_name(sha)) - rescue Rugged::ReferenceError - false - end - end - - def tag_names - cache.fetch(:tag_names) { raw_repository.tag_names } - end - - def commit_count - cache.fetch(:commit_count) do - begin - raw_repository.commit_count(self.root_ref) - rescue - 0 - end - end - end - - def branch_count - @branch_count ||= cache.fetch(:branch_count) { branches.size } - end - - def tag_count - @tag_count ||= cache.fetch(:tag_count) { raw_repository.rugged.tags.count } - end - - # Return repo size in megabytes - # Cached in redis - def size - cache.fetch(:size) { raw_repository.size } + ref_exists?(keep_around_ref_name(sha)) end def diverging_commit_counts(branch) @@ -320,48 +295,55 @@ class Repository end end - # Keys for data that can be affected for any commit push. - def cache_keys - %i(size commit_count - readme version contribution_guide changelog - license_blob license_key gitignore koding_yml) + def expire_tags_cache + expire_method_caches(%i(tag_names tag_count)) + @tags = nil end - # Keys for data on branch/tag operations. - def cache_keys_for_branches_and_tags - %i(branch_names tag_names branch_count tag_count) + def expire_branches_cache + expire_method_caches(%i(branch_names branch_count)) + @local_branches = nil end - def build_cache - (cache_keys + cache_keys_for_branches_and_tags).each do |key| - unless cache.exist?(key) - send(key) - end - end + def expire_statistics_caches + expire_method_caches(%i(size commit_count)) end - def expire_tags_cache - cache.expire(:tag_names) - @tags = nil + def expire_all_method_caches + expire_method_caches(CACHED_METHODS) end - def expire_branches_cache - cache.expire(:branch_names) - @branch_names = nil - @local_branches = nil + # Expires the caches of a specific set of methods + def expire_method_caches(methods) + methods.each do |key| + cache.expire(key) + + ivar = cache_instance_variable_name(key) + + remove_instance_variable(ivar) if instance_variable_defined?(ivar) + end end - def expire_cache(branch_name = nil, revision = nil) - cache_keys.each do |key| - cache.expire(key) + def expire_avatar_cache + expire_method_caches(%i(avatar)) + end + + # Refreshes the method caches of this repository. + # + # types - An Array of file types (e.g. `:readme`) used to refresh extra + # caches. + def refresh_method_caches(types) + to_refresh = [] + + types.each do |type| + methods = METHOD_CACHES_FOR_FILE_TYPES[type.to_sym] + + to_refresh.concat(Array(methods)) if methods end - expire_branch_cache(branch_name) - expire_avatar_cache(branch_name, revision) + expire_method_caches(to_refresh) - # This ensures this particular cache is flushed after the first commit to a - # new repository. - expire_emptiness_caches if empty? + to_refresh.each { |method| send(method) } end def expire_branch_cache(branch_name = nil) @@ -380,68 +362,32 @@ class Repository end def expire_root_ref_cache - cache.expire(:root_ref) - @root_ref = nil + expire_method_caches(%i(root_ref)) end # Expires the cache(s) used to determine if a repository is empty or not. def expire_emptiness_caches - cache.expire(:empty?) - @empty = nil - - expire_has_visible_content_cache - end - - def expire_has_visible_content_cache - cache.expire(:has_visible_content?) - @has_visible_content = nil - end + return unless empty? - def expire_branch_count_cache - cache.expire(:branch_count) - @branch_count = nil - end - - def expire_tag_count_cache - cache.expire(:tag_count) - @tag_count = nil + expire_method_caches(%i(empty?)) end def lookup_cache @lookup_cache ||= {} end - def expire_avatar_cache(branch_name = nil, revision = nil) - # Avatars are pulled from the default branch, thus if somebody pushes to a - # different branch there's no need to expire anything. - return if branch_name && branch_name != root_ref - - # We don't want to flush the cache if the commit didn't actually make any - # changes to any of the possible avatar files. - if revision && commit = self.commit(revision) - return unless commit.raw_diffs(deltas_only: true). - any? { |diff| AVATAR_FILES.include?(diff.new_path) } - end - - cache.expire(:avatar) - - @avatar = nil - end - def expire_exists_cache - cache.expire(:exists?) - @exists = nil + expire_method_caches(%i(exists?)) end # expire cache that doesn't depend on repository data (when expiring) def expire_content_cache expire_tags_cache - expire_tag_count_cache expire_branches_cache - expire_branch_count_cache expire_root_ref_cache expire_emptiness_caches expire_exists_cache + expire_statistics_caches end # Runs code after a repository has been created. @@ -456,9 +402,8 @@ class Repository # Runs code just before a repository is deleted. def before_delete expire_exists_cache - - expire_cache if exists? - + expire_all_method_caches + expire_branch_cache if exists? expire_content_cache repository_event(:remove_repository) @@ -475,9 +420,9 @@ class Repository # Runs code before pushing (= creating or removing) a tag. def before_push_tag - expire_cache + expire_statistics_caches + expire_emptiness_caches expire_tags_cache - expire_tag_count_cache repository_event(:push_tag) end @@ -485,7 +430,7 @@ class Repository # Runs code before removing a tag. def before_remove_tag expire_tags_cache - expire_tag_count_cache + expire_statistics_caches repository_event(:remove_tag) end @@ -494,15 +439,22 @@ class Repository expire_content_cache end + # Runs code after the HEAD of a repository is changed. + def after_change_head + expire_method_caches(METHOD_CACHES_FOR_FILE_TYPES.keys) + end + # Runs code after a repository has been forked/imported. def after_import expire_content_cache - build_cache + expire_tags_cache + expire_branches_cache end # Runs code after a new commit has been pushed. - def after_push_commit(branch_name, revision) - expire_cache(branch_name, revision) + def after_push_commit(branch_name) + expire_statistics_caches + expire_branch_cache(branch_name) repository_event(:push_commit, branch: branch_name) end @@ -510,8 +462,6 @@ class Repository # Runs code after a new branch has been created. def after_create_branch expire_branches_cache - expire_has_visible_content_cache - expire_branch_count_cache repository_event(:push_branch) end @@ -525,8 +475,6 @@ class Repository # Runs code after an existing branch has been removed. def after_remove_branch - expire_has_visible_content_cache - expire_branch_count_cache expire_branches_cache end @@ -553,86 +501,127 @@ class Repository Gitlab::Git::Blob.raw(self, oid) end + def root_ref + if raw_repository + raw_repository.root_ref + else + # When the repo does not exist we raise this error so no data is cached. + raise Rugged::ReferenceError + end + end + cache_method :root_ref + + def exists? + refs_directory_exists? + end + cache_method :exists? + + def empty? + raw_repository.empty? + end + cache_method :empty? + + # The size of this repository in megabytes. + def size + exists? ? raw_repository.size : 0.0 + end + cache_method :size, fallback: 0.0 + + def commit_count + root_ref ? raw_repository.commit_count(root_ref) : 0 + end + cache_method :commit_count, fallback: 0 + + def branch_names + branches.map(&:name) + end + cache_method :branch_names, fallback: [] + + def tag_names + raw_repository.tag_names + end + cache_method :tag_names, fallback: [] + + def branch_count + branches.size + end + cache_method :branch_count, fallback: 0 + + def tag_count + raw_repository.rugged.tags.count + end + cache_method :tag_count, fallback: 0 + + def avatar + if tree = file_on_head(:avatar) + tree.path + end + end + cache_method :avatar + def readme - cache.fetch(:readme) { tree(:head).readme } + if head = tree(:head) + head.readme + end end + cache_method :readme def version - cache.fetch(:version) do - tree(:head).blobs.find do |file| - file.name.casecmp('version').zero? - end - end + file_on_head(:version) end + cache_method :version def contribution_guide - cache.fetch(:contribution_guide) do - tree(:head).blobs.find do |file| - file.contributing? - end - end + file_on_head(:contributing) end + cache_method :contribution_guide def changelog - cache.fetch(:changelog) do - file_on_head(/\A(changelog|history|changes|news)/i) - end + file_on_head(:changelog) end + cache_method :changelog def license_blob - return nil unless head_exists? - - cache.fetch(:license_blob) do - file_on_head(/\A(licen[sc]e|copying)(\..+|\z)/i) - end + file_on_head(:license) end + cache_method :license_blob def license_key - return nil unless head_exists? + return unless exists? - cache.fetch(:license_key) do - Licensee.license(path).try(:key) - end + Licensee.license(path).try(:key) end + cache_method :license_key def gitignore - return nil if !exists? || empty? - - cache.fetch(:gitignore) do - file_on_head(/\A\.gitignore\z/) - end + file_on_head(:gitignore) end + cache_method :gitignore def koding_yml - return nil unless head_exists? - - cache.fetch(:koding_yml) do - file_on_head(/\A\.koding\.yml\z/) - end + file_on_head(:koding) end + cache_method :koding_yml def gitlab_ci_yml - return nil unless head_exists? - - @gitlab_ci_yml ||= tree(:head).blobs.find do |file| - file.name == '.gitlab-ci.yml' - end - rescue Rugged::ReferenceError - # For unknow reason spinach scenario "Scenario: I change project path" - # lead to "Reference 'HEAD' not found" exception from Repository#empty? - nil + file_on_head(:gitlab_ci) end + cache_method :gitlab_ci_yml def head_commit @head_commit ||= commit(self.root_ref) end def head_tree - @head_tree ||= Tree.new(self, head_commit.sha, nil) + if head_commit + @head_tree ||= Tree.new(self, head_commit.sha, nil) + end end - def tree(sha = :head, path = nil) + def tree(sha = :head, path = nil, recursive: false) if sha == :head + return unless head_commit + if path.nil? return head_tree else @@ -640,7 +629,7 @@ class Repository end end - Tree.new(self, sha, path) + Tree.new(self, sha, path, recursive: recursive) end def blob_at_branch(branch_name, path) @@ -670,11 +659,19 @@ class Repository end def last_commit_for_path(sha, path) - args = %W(#{Gitlab.config.git.bin_path} rev-list --max-count=1 #{sha} -- #{path}) - sha = Gitlab::Popen.popen(args, path_to_repo).first.strip + sha = last_commit_id_for_path(sha, path) commit(sha) end + def last_commit_id_for_path(sha, path) + key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}" + + cache.fetch(key) do + args = %W(#{Gitlab.config.git.bin_path} rev-list --max-count=1 #{sha} -- #{path}) + Gitlab::Popen.popen(args, path_to_repo).first.strip + end + end + def next_branch(name, opts = {}) branch_ids = self.branch_names.map do |n| next 1 if n == name @@ -782,10 +779,6 @@ class Repository @tags ||= raw_repository.tags end - def root_ref - @root_ref ||= cache.fetch(:root_ref) { raw_repository.root_ref } - end - def commit_dir(user, path, message, branch, author_email: nil, author_name: nil) update_branch_with_hooks(user, branch) do |ref| options = { @@ -970,7 +963,7 @@ class Repository update_branch_with_hooks(user, base_branch) do committer = user_to_committer(user) source_sha = Rugged::Commit.create(rugged, - message: commit.revert_message, + message: commit.revert_message(user), author: committer, committer: committer, tree: revert_tree_id, @@ -1063,12 +1056,25 @@ class Repository merge_base(ancestor_id, descendant_id) == ancestor_id end - def search_files(query, ref) + def empty_repo? + !exists? || !has_visible_content? + end + + def search_files_by_content(query, ref) + return [] if empty_repo? || query.blank? + offset = 2 args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/) end + def search_files_by_name(query, ref) + return [] if empty_repo? || query.blank? + + args = %W(#{Gitlab.config.git.bin_path} ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)}) + Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:strip) + end + def fetch_ref(source_path, source_ref, target_ref) args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) Gitlab::Popen.popen(args, path_to_repo) @@ -1130,28 +1136,55 @@ class Repository end end - def avatar - return nil unless exists? + # Caches the supplied block both in a cache and in an instance variable. + # + # The cache key and instance variable are named the same way as the value of + # the `key` argument. + # + # This method will return `nil` if the corresponding instance variable is also + # set to `nil`. This ensures we don't keep yielding the block when it returns + # `nil`. + # + # key - The name of the key to cache the data in. + # fallback - A value to fall back to in the event of a Git error. + def cache_method_output(key, fallback: nil, &block) + ivar = cache_instance_variable_name(key) + + if instance_variable_defined?(ivar) + instance_variable_get(ivar) + else + begin + instance_variable_set(ivar, cache.fetch(key, &block)) + rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository + # if e.g. HEAD or the entire repository doesn't exist we want to + # gracefully handle this and not cache anything. + fallback + end + end + end + + def cache_instance_variable_name(key) + :"@#{key.to_s.tr('?!', '')}" + end - @avatar ||= cache.fetch(:avatar) do - AVATAR_FILES.find do |file| - blob_at_branch(root_ref, file) + def file_on_head(type) + if head = tree(:head) + head.blobs.find do |file| + Gitlab::FileDetector.type_of(file.name) == type end end end private - def cache - @cache ||= RepositoryCache.new(path_with_namespace, @project.id) - end + def refs_directory_exists? + return false unless path_with_namespace - def head_exists? - exists? && !empty? && !rugged.head_unborn? + File.exist?(File.join(path_to_repo, 'refs')) end - def file_on_head(regex) - tree(:head).blobs.find { |file| file.name =~ regex } + def cache + @cache ||= RepositoryCache.new(path_with_namespace, @project.id) end def tags_sorted_by_committed_date diff --git a/app/models/route.rb b/app/models/route.rb new file mode 100644 index 00000000000..caf596efa79 --- /dev/null +++ b/app/models/route.rb @@ -0,0 +1,22 @@ +class Route < ActiveRecord::Base + belongs_to :source, polymorphic: true + + validates :source, presence: true + + validates :path, + length: { within: 1..255 }, + presence: true, + uniqueness: { case_sensitive: false } + + after_update :rename_children, if: :path_changed? + + def rename_children + # We update each row separately because MySQL does not have regexp_replace. + # rubocop:disable Rails/FindEach + Route.where('path LIKE ?', "#{path_was}/%").each do |route| + # Note that update column skips validation and callbacks. + # We need this to avoid recursive call of rename_children method + route.update_column(:path, route.path.sub(path_was, path)) + end + end +end diff --git a/app/models/service.rb b/app/models/service.rb index 625fbc48302..043be222f3a 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -8,6 +8,7 @@ class Service < ActiveRecord::Base default_value_for :push_events, true default_value_for :issues_events, true default_value_for :confidential_issues_events, true + default_value_for :commit_events, true default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true @@ -75,6 +76,11 @@ class Service < ActiveRecord::Base def to_param # implement inside child + self.class.to_param + end + + def self.to_param + raise NotImplementedError end def fields @@ -91,7 +97,11 @@ class Service < ActiveRecord::Base end def event_names - supported_events.map { |event| "#{event}_events" } + self.class.event_names + end + + def self.event_names + self.supported_events.map { |event| "#{event}_events" } end def event_field(event) @@ -103,6 +113,10 @@ class Service < ActiveRecord::Base end def supported_events + self.class.supported_events + end + + def self.supported_events %w(push tag_push issue confidential_issue merge_request wiki_page) end @@ -202,7 +216,6 @@ class Service < ActiveRecord::Base bamboo buildkite builds_email - pipelines_email bugzilla campfire custom_issue_tracker @@ -214,19 +227,24 @@ class Service < ActiveRecord::Base hipchat irker jira + kubernetes + mattermost_slash_commands + mattermost + pipelines_email pivotaltracker pushover redmine + slack_slash_commands slack teamcity ] end - def self.create_from_template(project_id, template) + def self.build_from_template(project_id, template) service = template.dup service.template = false service.project_id = project_id - service if service.save + service end private diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 2373b445009..771a7350556 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -6,6 +6,7 @@ class Snippet < ActiveRecord::Base include Referable include Sortable include Awardable + include Mentionable cache_markdown_field :title, pipeline: :single_line cache_markdown_field :content @@ -26,9 +27,9 @@ class Snippet < ActiveRecord::Base delegate :name, :email, to: :author, prefix: true, allow_nil: true validates :author, presence: true - validates :title, presence: true, length: { within: 0..255 } + validates :title, presence: true, length: { maximum: 255 } validates :file_name, - length: { within: 0..255 }, + length: { maximum: 255 }, format: { with: Gitlab::Regex.file_name_regex, message: Gitlab::Regex.file_name_regex_message } @@ -63,14 +64,14 @@ class Snippet < ActiveRecord::Base @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) end - def to_reference(from_project = nil) + def to_reference(from_project = nil, full: false) reference = "#{self.class.reference_prefix}#{id}" - if cross_project_reference?(from_project) - reference = project.to_reference + reference + if project.present? + "#{project.to_reference(from_project, full: full)}#{reference}" + else + reference end - - reference end def self.content_types @@ -93,6 +94,10 @@ class Snippet < ActiveRecord::Base 0 end + def file_name + super.to_s + end + # alias for compatibility with blobs and highlighting def path file_name diff --git a/app/models/subscription.rb b/app/models/subscription.rb index 3b8aa1eb866..17869c8bac2 100644 --- a/app/models/subscription.rb +++ b/app/models/subscription.rb @@ -1,8 +1,9 @@ class Subscription < ActiveRecord::Base belongs_to :user + belongs_to :project belongs_to :subscribable, polymorphic: true - validates :user_id, - uniqueness: { scope: [:subscribable_id, :subscribable_type] }, - presence: true + validates :user, :subscribable, presence: true + + validates :project_id, uniqueness: { scope: [:subscribable_id, :subscribable_type, :user_id] } end diff --git a/app/models/timelog.rb b/app/models/timelog.rb new file mode 100644 index 00000000000..f768c4e3da5 --- /dev/null +++ b/app/models/timelog.rb @@ -0,0 +1,6 @@ +class Timelog < ActiveRecord::Base + validates :time_spent, :user, presence: true + + belongs_to :trackable, polymorphic: true + belongs_to :user +end diff --git a/app/models/todo.rb b/app/models/todo.rb index f5ade1cc293..4c99aa0d3be 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -6,13 +6,15 @@ class Todo < ActiveRecord::Base BUILD_FAILED = 3 MARKED = 4 APPROVAL_REQUIRED = 5 # This is an EE-only feature + UNMERGEABLE = 6 ACTION_NAMES = { ASSIGNED => :assigned, MENTIONED => :mentioned, BUILD_FAILED => :build_failed, MARKED => :marked, - APPROVAL_REQUIRED => :approval_required + APPROVAL_REQUIRED => :approval_required, + UNMERGEABLE => :unmergeable } belongs_to :author, class_name: "User" @@ -66,6 +68,10 @@ class Todo < ActiveRecord::Base end end + def unmergeable? + action == UNMERGEABLE + end + def build_failed? action == BUILD_FAILED end diff --git a/app/models/tree.rb b/app/models/tree.rb index 7c4ed6e393b..fe148b0ec65 100644 --- a/app/models/tree.rb +++ b/app/models/tree.rb @@ -3,21 +3,24 @@ class Tree attr_accessor :repository, :sha, :path, :entries - def initialize(repository, sha, path = '/') + def initialize(repository, sha, path = '/', recursive: false) path = '/' if path.blank? @repository = repository @sha = sha @path = path + @recursive = recursive git_repo = @repository.raw_repository - @entries = Gitlab::Git::Tree.where(git_repo, @sha, @path) + @entries = get_entries(git_repo, @sha, @path, recursive: @recursive) end def readme return @readme if defined?(@readme) - available_readmes = blobs.select(&:readme?) + available_readmes = blobs.select do |blob| + Gitlab::FileDetector.type_of(blob.name) == :readme + end previewable_readmes = available_readmes.select do |blob| previewable?(blob.name) @@ -58,4 +61,21 @@ class Tree def sorted_entries trees + blobs + submodules end + + private + + def get_entries(git_repo, sha, path, recursive: false) + current_path_entries = Gitlab::Git::Tree.where(git_repo, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(get_entries(git_repo, sha, entry.path, recursive: true)) + end + end + + ordered_entries + end end diff --git a/app/models/user.rb b/app/models/user.rb index af3c0b7dc02..06dd98a3188 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -13,6 +13,7 @@ class User < ActiveRecord::Base DEFAULT_NOTIFICATION_LEVEL = :participating add_authentication_token_field :authentication_token + add_authentication_token_field :incoming_email_token default_value_for :admin, false default_value_for(:external) { current_application_settings.user_default_external } @@ -55,6 +56,7 @@ class User < ActiveRecord::Base has_many :personal_access_tokens, dependent: :destroy has_many :identities, dependent: :destroy, autosave: true has_many :u2f_registrations, dependent: :destroy + has_many :chat_names, dependent: :destroy # Groups has_many :members, dependent: :destroy @@ -71,6 +73,8 @@ class User < ActiveRecord::Base has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy has_many :starred_projects, through: :users_star_projects, source: :project + has_many :project_authorizations + has_many :authorized_projects, through: :project_authorizations, source: :project has_many :snippets, dependent: :destroy, foreign_key: :author_id has_many :issues, dependent: :destroy, foreign_key: :author_id @@ -95,6 +99,7 @@ class User < ActiveRecord::Base # # Note: devise :validatable above adds validations for :email and :password validates :name, presence: true + validates_confirmation_of :email validates :notification_email, presence: true validates :notification_email, email: true, if: ->(user) { user.notification_email != user.email } validates :public_email, presence: true, uniqueness: true, email: true, allow_blank: true @@ -119,7 +124,7 @@ class User < ActiveRecord::Base before_validation :set_public_email, if: ->(user) { user.public_email_changed? } after_update :update_emails_with_primary_email, if: ->(user) { user.email_changed? } - before_save :ensure_authentication_token + before_save :ensure_authentication_token, :ensure_incoming_email_token before_save :ensure_external_user_rights after_save :ensure_namespace_correct after_initialize :set_projects_limit @@ -172,7 +177,10 @@ class User < ActiveRecord::Base scope :external, -> { where(external: true) } scope :active, -> { with_state(:active) } scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all } - scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members)') } + scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') } + scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) } + scope :order_recent_sign_in, -> { reorder(last_sign_in_at: :desc) } + scope :order_oldest_sign_in, -> { reorder(last_sign_in_at: :asc) } def self.with_two_factor joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id"). @@ -200,8 +208,8 @@ class User < ActiveRecord::Base def sort(method) case method.to_s - when 'recent_sign_in' then reorder(last_sign_in_at: :desc) - when 'oldest_sign_in' then reorder(last_sign_in_at: :asc) + when 'recent_sign_in' then order_recent_sign_in + when 'oldest_sign_in' then order_oldest_sign_in else order_by(method) end @@ -224,19 +232,19 @@ class User < ActiveRecord::Base def filter(filter_name) case filter_name when 'admins' - self.admins + admins when 'blocked' - self.blocked + blocked when 'two_factor_disabled' - self.without_two_factor + without_two_factor when 'two_factor_enabled' - self.with_two_factor + with_two_factor when 'wop' - self.without_projects + without_projects when 'external' - self.external + external else - self.active + active end end @@ -286,8 +294,12 @@ class User < ActiveRecord::Base end end + def find_by_username(username) + iwhere(username: username).take + end + def find_by_username!(username) - find_by!('lower(username) = ?', username.downcase) + iwhere(username: username).take! end def find_by_personal_access_token(token_string) @@ -295,19 +307,11 @@ class User < ActiveRecord::Base personal_access_token.user if personal_access_token end - def by_username_or_id(name_or_id) - find_by('users.username = ? OR users.id = ?', name_or_id.to_s, name_or_id.to_i) - end - # Returns a user for the given SSH key. def find_by_ssh_key_id(key_id) find_by(id: Key.unscoped.select(:user_id).where(id: key_id)) end - def build_user(attrs = {}) - User.new(attrs) - end - def reference_prefix '@' end @@ -329,12 +333,12 @@ class User < ActiveRecord::Base username end - def to_reference(_from_project = nil, _target_project = nil) + def to_reference(_from_project = nil, target_project: nil, full: nil) "#{self.class.reference_prefix}#{username}" end def generate_password - if self.force_random_password + if force_random_password self.password = self.password_confirmation = Devise.friendly_token.first(Devise.password_length.min) end end @@ -375,56 +379,55 @@ class User < ActiveRecord::Base end def two_factor_otp_enabled? - self.otp_required_for_login? + otp_required_for_login? end def two_factor_u2f_enabled? - self.u2f_registrations.exists? + u2f_registrations.exists? end def namespace_uniq # Return early if username already failed the first uniqueness validation - return if self.errors.key?(:username) && - self.errors[:username].include?('has already been taken') + return if errors.key?(:username) && + errors[:username].include?('has already been taken') - namespace_name = self.username - existing_namespace = Namespace.by_path(namespace_name) - if existing_namespace && existing_namespace != self.namespace - self.errors.add(:username, 'has already been taken') + existing_namespace = Namespace.by_path(username) + if existing_namespace && existing_namespace != namespace + errors.add(:username, 'has already been taken') end end def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" + unless avatar.image? + errors.add :avatar, "only images allowed" end end def unique_email - if !self.emails.exists?(email: self.email) && Email.exists?(email: self.email) - self.errors.add(:email, 'has already been taken') + if !emails.exists?(email: email) && Email.exists?(email: email) + errors.add(:email, 'has already been taken') end end def owns_notification_email - return if self.temp_oauth_email? + return if temp_oauth_email? - self.errors.add(:notification_email, "is not an email you own") unless self.all_emails.include?(self.notification_email) + errors.add(:notification_email, "is not an email you own") unless all_emails.include?(notification_email) end def owns_public_email - return if self.public_email.blank? + return if public_email.blank? - self.errors.add(:public_email, "is not an email you own") unless self.all_emails.include?(self.public_email) + errors.add(:public_email, "is not an email you own") unless all_emails.include?(public_email) end def update_emails_with_primary_email - primary_email_record = self.emails.find_by(email: self.email) + primary_email_record = emails.find_by(email: email) if primary_email_record primary_email_record.destroy - self.emails.create(email: self.email_was) + emails.create(email: email_was) - self.update_secondary_emails! + update_secondary_emails! end end @@ -436,16 +439,48 @@ class User < ActiveRecord::Base Group.where("namespaces.id IN (#{union.to_sql})") end - # Returns projects user is authorized to access. - # - # If you change the logic of this method, please also update `Project#authorized_for_user` + def refresh_authorized_projects + Users::RefreshAuthorizedProjectsService.new(self).execute + end + + def remove_project_authorizations(project_ids) + project_authorizations.where(project_id: project_ids).delete_all + end + + def set_authorized_projects_column + unless authorized_projects_populated + update_column(:authorized_projects_populated, true) + end + end + def authorized_projects(min_access_level = nil) - Project.where("projects.id IN (#{projects_union(min_access_level).to_sql})") + refresh_authorized_projects unless authorized_projects_populated + + # We're overriding an association, so explicitly call super with no arguments or it would be passed as `force_reload` to the association + projects = super() + projects = projects.where('project_authorizations.access_level >= ?', min_access_level) if min_access_level + + projects + end + + def authorized_project?(project, min_access_level = nil) + authorized_projects(min_access_level).exists?({ id: project.id }) + end + + # Returns the projects this user has reporter (or greater) access to, limited + # to at most the given projects. + # + # This method is useful when you have a list of projects and want to + # efficiently check to which of these projects the user has at least reporter + # access. + def projects_with_reporter_access_limited_to(projects) + authorized_projects(Gitlab::Access::REPORTER).where(id: projects) end def viewable_starred_projects - starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (#{projects_union.to_sql})", - [Project::PUBLIC, Project::INTERNAL]) + starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (?)", + [Project::PUBLIC, Project::INTERNAL], + authorized_projects.select(:project_id)) end def owned_projects @@ -466,7 +501,7 @@ class User < ActiveRecord::Base end def require_ssh_key? - keys.count == 0 + keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh') end def require_password? @@ -568,7 +603,7 @@ class User < ActiveRecord::Base end def project_deploy_keys - DeployKey.unscoped.in_projects(self.authorized_projects.pluck(:id)).distinct(:id) + DeployKey.unscoped.in_projects(authorized_projects.pluck(:id)).distinct(:id) end def accessible_deploy_keys @@ -584,38 +619,38 @@ class User < ActiveRecord::Base end def sanitize_attrs - %w(name username skype linkedin twitter).each do |attr| - value = self.send(attr) - self.send("#{attr}=", Sanitize.clean(value)) if value.present? + %w[name username skype linkedin twitter].each do |attr| + value = public_send(attr) + public_send("#{attr}=", Sanitize.clean(value)) if value.present? end end def set_notification_email - if self.notification_email.blank? || !self.all_emails.include?(self.notification_email) - self.notification_email = self.email + if notification_email.blank? || !all_emails.include?(notification_email) + self.notification_email = email end end def set_public_email - if self.public_email.blank? || !self.all_emails.include?(self.public_email) + if public_email.blank? || !all_emails.include?(public_email) self.public_email = '' end end def update_secondary_emails! - self.set_notification_email - self.set_public_email - self.save if self.notification_email_changed? || self.public_email_changed? + set_notification_email + set_public_email + save if notification_email_changed? || public_email_changed? end def set_projects_limit # `User.select(:id)` raises # `ActiveModel::MissingAttributeError: missing attribute: projects_limit` # without this safeguard! - return unless self.has_attribute?(:projects_limit) + return unless has_attribute?(:projects_limit) connection_default_value_defined = new_record? && !projects_limit_changed? - return unless self.projects_limit.nil? || connection_default_value_defined + return unless projects_limit.nil? || connection_default_value_defined self.projects_limit = current_application_settings.default_projects_limit end @@ -645,7 +680,7 @@ class User < ActiveRecord::Base def with_defaults User.defaults.each do |k, v| - self.send("#{k}=", v) + public_send("#{k}=", v) end self @@ -656,20 +691,6 @@ class User < ActiveRecord::Base project.project_member(self) end - # Reset project events cache related to this user - # - # Since we do cache @event we need to reset cache in special cases: - # * when the user changes their avatar - # Events cache stored like events/23-20130109142513. - # The cache key includes updated_at timestamp. - # Thus it will automatically generate a new fragment - # when the event is updated because the key changes. - def reset_events_cache - Event.where(author_id: self.id). - order('id DESC').limit(1000). - update_all(updated_at: Time.now) - end - def full_website_url return "http://#{website_url}" if website_url !~ /\Ahttps?:\/\// @@ -698,8 +719,8 @@ class User < ActiveRecord::Base def all_emails all_emails = [] - all_emails << self.email unless self.temp_oauth_email? - all_emails.concat(self.emails.map(&:email)) + all_emails << email unless temp_oauth_email? + all_emails.concat(emails.map(&:email)) all_emails end @@ -713,21 +734,21 @@ class User < ActiveRecord::Base def ensure_namespace_correct # Ensure user has namespace - self.create_namespace!(path: self.username, name: self.username) unless self.namespace + create_namespace!(path: username, name: username) unless namespace - if self.username_changed? - self.namespace.update_attributes(path: self.username, name: self.username) + if username_changed? + namespace.update_attributes(path: username, name: username) end end def post_create_hook - log_info("User \"#{self.name}\" (#{self.email}) was created") - notification_service.new_user(self, @reset_token) if self.created_by_id + log_info("User \"#{name}\" (#{email}) was created") + notification_service.new_user(self, @reset_token) if created_by_id system_hook_service.execute_hooks_for(self, :create) end def post_destroy_hook - log_info("User \"#{self.name}\" (#{self.email}) was removed") + log_info("User \"#{name}\" (#{email}) was removed") system_hook_service.execute_hooks_for(self, :destroy) end @@ -771,7 +792,7 @@ class User < ActiveRecord::Base end def oauth_authorized_tokens - Doorkeeper::AccessToken.where(resource_owner_id: self.id, revoked_at: nil) + Doorkeeper::AccessToken.where(resource_owner_id: id, revoked_at: nil) end # Returns the projects a user contributed to in the last year. @@ -875,20 +896,6 @@ class User < ActiveRecord::Base private - def projects_union(min_access_level = nil) - relations = [personal_projects.select(:id), - groups_projects.select(:id), - projects.select(:id), - groups.joins(:shared_projects).select(:project_id)] - - if min_access_level - scope = { access_level: Gitlab::Access.all_values.select { |access| access >= min_access_level } } - relations = [relations.shift] + relations.map { |relation| relation.where(members: scope) } - end - - Gitlab::SQL::Union.new(relations) - end - def ci_projects_union scope = { access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] } groups = groups_projects.where(members: scope) @@ -904,7 +911,7 @@ class User < ActiveRecord::Base end def ensure_external_user_rights - return unless self.external? + return unless external? self.can_create_group = false self.projects_limit = 0 @@ -916,7 +923,7 @@ class User < ActiveRecord::Base if current_application_settings.domain_blacklist_enabled? blocked_domains = current_application_settings.domain_blacklist - if domain_matches?(blocked_domains, self.email) + if domain_matches?(blocked_domains, email) error = 'is not from an allowed domain.' valid = false end @@ -924,7 +931,7 @@ class User < ActiveRecord::Base allowed_domains = current_application_settings.domain_whitelist unless allowed_domains.blank? - if domain_matches?(allowed_domains, self.email) + if domain_matches?(allowed_domains, email) valid = true else error = "domain is not authorized for sign-up" @@ -932,7 +939,7 @@ class User < ActiveRecord::Base end end - self.errors.add(:email, error) unless valid + errors.add(:email, error) unless valid valid end @@ -945,4 +952,13 @@ class User < ActiveRecord::Base signup_domain =~ regexp end end + + def generate_token(token_field) + if token_field == :incoming_email_token + # Needs to be all lowercase and alphanumeric because it's gonna be used in an email address. + SecureRandom.hex.to_i(16).to_s(36) + else + super + end + end end |