diff options
Diffstat (limited to 'app/models')
96 files changed, 3300 insertions, 1576 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index d9113ffd99a..fa8f8bc3a5f 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -1,34 +1,5 @@ class Ability class << self - # rubocop: disable Metrics/CyclomaticComplexity - def allowed(user, subject) - return anonymous_abilities(user, subject) if user.nil? - return [] unless user.is_a?(User) - return [] if user.blocked? - - abilities_by_subject_class(user: user, subject: subject) - end - - def abilities_by_subject_class(user:, subject:) - case subject - when CommitStatus then commit_status_abilities(user, subject) - when Project then project_abilities(user, subject) - when Issue then issue_abilities(user, subject) - when Note then note_abilities(user, subject) - when ProjectSnippet then project_snippet_abilities(user, subject) - when PersonalSnippet then personal_snippet_abilities(user, subject) - when MergeRequest then merge_request_abilities(user, subject) - when Group then group_abilities(user, subject) - when Namespace then namespace_abilities(user, subject) - when GroupMember then group_member_abilities(user, subject) - when ProjectMember then project_member_abilities(user, subject) - when User then user_abilities - when ExternalIssue, Deployment, Environment then project_abilities(user, subject.project) - when Ci::Runner then runner_abilities(user, subject) - else [] - end.concat(global_abilities(user)) - end - # Given a list of users and a project this method returns the users that can # read the given project. def users_that_can_read_project(users, project) @@ -61,347 +32,7 @@ class Ability issues.select { |issue| issue.visible_to_user?(user) } end - # List of possible abilities for anonymous user - def anonymous_abilities(user, subject) - if subject.is_a?(PersonalSnippet) - anonymous_personal_snippet_abilities(subject) - elsif subject.is_a?(ProjectSnippet) - anonymous_project_snippet_abilities(subject) - elsif subject.is_a?(CommitStatus) - anonymous_commit_status_abilities(subject) - elsif subject.is_a?(Project) || subject.respond_to?(:project) - anonymous_project_abilities(subject) - elsif subject.is_a?(Group) || subject.respond_to?(:group) - anonymous_group_abilities(subject) - elsif subject.is_a?(User) - anonymous_user_abilities - else - [] - end - end - - def anonymous_project_abilities(subject) - project = if subject.is_a?(Project) - subject - else - subject.project - end - - if project && project.public? - rules = [ - :read_project, - :read_wiki, - :read_label, - :read_milestone, - :read_project_snippet, - :read_project_member, - :read_merge_request, - :read_note, - :read_pipeline, - :read_commit_status, - :read_container_image, - :download_code - ] - - # Allow to read builds by anonymous user if guests are allowed - rules << :read_build if project.public_builds? - - # Allow to read issues by anonymous user if issue is not confidential - rules << :read_issue unless subject.is_a?(Issue) && subject.confidential? - - rules - project_disabled_features_rules(project) - else - [] - end - end - - def anonymous_commit_status_abilities(subject) - rules = anonymous_project_abilities(subject.project) - # If subject is Ci::Build which inherits from CommitStatus filter the abilities - rules = filter_build_abilities(rules) if subject.is_a?(Ci::Build) - rules - end - - def anonymous_group_abilities(subject) - rules = [] - - group = if subject.is_a?(Group) - subject - else - subject.group - end - - rules << :read_group if group.public? - - rules - end - - def anonymous_personal_snippet_abilities(snippet) - if snippet.public? - [:read_personal_snippet] - else - [] - end - end - - def anonymous_project_snippet_abilities(snippet) - if snippet.public? - [:read_project_snippet] - else - [] - end - end - - def anonymous_user_abilities - [:read_user] unless restricted_public_level? - end - - def global_abilities(user) - rules = [] - rules << :create_group if user.can_create_group - rules << :read_users_list - rules - end - - def project_abilities(user, project) - rules = [] - key = "/user/#{user.id}/project/#{project.id}" - - RequestStore.store[key] ||= begin - # Push abilities on the users team role - rules.push(*project_team_rules(project.team, user)) - - owner = user.admin? || - project.owner == user || - (project.group && project.group.has_owner?(user)) - - if owner - rules.push(*project_owner_rules) - end - - if project.public? || (project.internal? && !user.external?) - rules.push(*public_project_rules) - - # Allow to read builds for internal projects - rules << :read_build if project.public_builds? - - unless owner || project.team.member?(user) || project_group_member?(project, user) - rules << :request_access if project.request_access_enabled - end - end - - if project.archived? - rules -= project_archived_rules - end - - rules - project_disabled_features_rules(project) - end - end - - def project_team_rules(team, user) - # Rules based on role in project - if team.master?(user) - project_master_rules - elsif team.developer?(user) - project_dev_rules - elsif team.reporter?(user) - project_report_rules - elsif team.guest?(user) - project_guest_rules - else - [] - end - end - - def public_project_rules - @public_project_rules ||= project_guest_rules + [ - :download_code, - :fork_project, - :read_commit_status, - :read_pipeline, - :read_container_image - ] - end - - def project_guest_rules - @project_guest_rules ||= [ - :read_project, - :read_wiki, - :read_issue, - :read_label, - :read_milestone, - :read_project_snippet, - :read_project_member, - :read_merge_request, - :read_note, - :create_project, - :create_issue, - :create_note, - :upload_file - ] - end - - def project_report_rules - @project_report_rules ||= project_guest_rules + [ - :download_code, - :fork_project, - :create_project_snippet, - :update_issue, - :admin_issue, - :admin_label, - :read_commit_status, - :read_build, - :read_container_image, - :read_pipeline, - :read_environment, - :read_deployment - ] - end - - def project_dev_rules - @project_dev_rules ||= project_report_rules + [ - :admin_merge_request, - :update_merge_request, - :create_commit_status, - :update_commit_status, - :create_build, - :update_build, - :create_pipeline, - :update_pipeline, - :create_merge_request, - :create_wiki, - :push_code, - :create_container_image, - :update_container_image, - :create_environment, - :create_deployment - ] - end - - def project_archived_rules - @project_archived_rules ||= [ - :create_merge_request, - :push_code, - :push_code_to_protected_branches, - :update_merge_request, - :admin_merge_request - ] - end - - def project_master_rules - @project_master_rules ||= project_dev_rules + [ - :push_code_to_protected_branches, - :update_project_snippet, - :update_environment, - :update_deployment, - :admin_milestone, - :admin_project_snippet, - :admin_project_member, - :admin_merge_request, - :admin_note, - :admin_wiki, - :admin_project, - :admin_commit_status, - :admin_build, - :admin_container_image, - :admin_pipeline, - :admin_environment, - :admin_deployment - ] - end - - def project_owner_rules - @project_owner_rules ||= project_master_rules + [ - :change_namespace, - :change_visibility_level, - :rename_project, - :remove_project, - :archive_project, - :remove_fork_project, - :destroy_merge_request, - :destroy_issue - ] - end - - def project_disabled_features_rules(project) - rules = [] - - unless project.issues_enabled - rules += named_abilities('issue') - end - - unless project.merge_requests_enabled - rules += named_abilities('merge_request') - end - - unless project.issues_enabled or project.merge_requests_enabled - rules += named_abilities('label') - rules += named_abilities('milestone') - end - - unless project.snippets_enabled - rules += named_abilities('project_snippet') - end - - unless project.wiki_enabled - rules += named_abilities('wiki') - end - - unless project.builds_enabled - rules += named_abilities('build') - rules += named_abilities('pipeline') - rules += named_abilities('environment') - rules += named_abilities('deployment') - end - - unless project.container_registry_enabled - rules += named_abilities('container_image') - end - - rules - end - - def group_abilities(user, group) - rules = [] - rules << :read_group if can_read_group?(user, group) - - owner = user.admin? || group.has_owner?(user) - master = owner || group.has_master?(user) - - # Only group masters and group owners can create new projects - if master - rules += [ - :create_projects, - :admin_milestones - ] - end - - # Only group owner and administrators can admin group - if owner - rules += [ - :admin_group, - :admin_namespace, - :admin_group_member, - :change_visibility_level - ] - end - - if group.public? || (group.internal? && !user.external?) - rules << :request_access if group.request_access_enabled && group.users.exclude?(user) - end - - rules.flatten - end - - def can_read_group?(user, group) - return true if user.admin? - return true if group.public? - return true if group.internal? && !user.external? - return true if group.users.include?(user) - - GroupProjectsFinder.new(group).execute(user).any? - end - + # TODO: make this private and use the actual abilities stuff for this def can_edit_note?(user, note) return false if !note.editable? || !user.present? return true if note.author == user || user.admin? @@ -414,202 +45,23 @@ class Ability end end - def namespace_abilities(user, namespace) - rules = [] - - # Only namespace owner and administrators can admin it - if namespace.owner == user || user.admin? - rules += [ - :create_projects, - :admin_namespace - ] - end - - rules.flatten - end - - [:issue, :merge_request].each do |name| - define_method "#{name}_abilities" do |user, subject| - rules = [] - - if subject.author == user || (subject.respond_to?(:assignee) && subject.assignee == user) - rules += [ - :"read_#{name}", - :"update_#{name}", - ] - end - - rules += project_abilities(user, subject.project) - rules = filter_confidential_issues_abilities(user, subject, rules) if subject.is_a?(Issue) - rules - end - end - - def note_abilities(user, note) - rules = [] - - if note.author == user - rules += [ - :read_note, - :update_note, - :admin_note - ] - end - - if note.respond_to?(:project) && note.project - rules += project_abilities(user, note.project) - end - - rules - end - - def personal_snippet_abilities(user, snippet) - rules = [] - - if snippet.author == user - rules += [ - :read_personal_snippet, - :update_personal_snippet, - :admin_personal_snippet - ] - end - - if snippet.public? || (snippet.internal? && !user.external?) - rules << :read_personal_snippet - end - - rules - end - - def project_snippet_abilities(user, snippet) - rules = [] - - if snippet.author == user || user.admin? - rules += [ - :read_project_snippet, - :update_project_snippet, - :admin_project_snippet - ] - end - - if snippet.public? || (snippet.internal? && !user.external?) || (snippet.private? && snippet.project.team.member?(user)) - rules << :read_project_snippet - end - - rules + def allowed?(user, action, subject) + allowed(user, subject).include?(action) end - def group_member_abilities(user, subject) - rules = [] - target_user = subject.user - group = subject.group - - unless group.last_owner?(target_user) - can_manage = group_abilities(user, group).include?(:admin_group_member) - - if can_manage - rules << :update_group_member - rules << :destroy_group_member - elsif user == target_user - rules << :destroy_group_member - end - end - - rules - end - - def project_member_abilities(user, subject) - rules = [] - target_user = subject.user - project = subject.project - - unless target_user == project.owner - can_manage = project_abilities(user, project).include?(:admin_project_member) - - if can_manage - rules << :update_project_member - rules << :destroy_project_member - elsif user == target_user - rules << :destroy_project_member - end - end - - rules - end - - def commit_status_abilities(user, subject) - rules = project_abilities(user, subject.project) - # If subject is Ci::Build which inherits from CommitStatus filter the abilities - rules = filter_build_abilities(rules) if subject.is_a?(Ci::Build) - rules - end - - def filter_build_abilities(rules) - # If we can't read build we should also not have that - # ability when looking at this in context of commit_status - %w(read create update admin).each do |rule| - rules.delete(:"#{rule}_commit_status") unless rules.include?(:"#{rule}_build") - end - rules - end - - def runner_abilities(user, runner) - if user.is_admin? - [:assign_runner] - elsif runner.is_shared? || runner.locked? - [] - elsif user.ci_authorized_runners.include?(runner) - [:assign_runner] - else - [] - end - end - - def user_abilities - [:read_user] - end + def allowed(user, subject) + return uncached_allowed(user, subject) unless RequestStore.active? - def abilities - @abilities ||= begin - abilities = Six.new - abilities << self - abilities - end + user_key = user ? user.id : 'anonymous' + subject_key = subject ? "#{subject.class.name}/#{subject.id}" : 'global' + key = "/ability/#{user_key}/#{subject_key}" + RequestStore[key] ||= uncached_allowed(user, subject).freeze end private - def restricted_public_level? - current_application_settings.restricted_visibility_levels.include?(Gitlab::VisibilityLevel::PUBLIC) - end - - def named_abilities(name) - [ - :"read_#{name}", - :"create_#{name}", - :"update_#{name}", - :"admin_#{name}" - ] - end - - def filter_confidential_issues_abilities(user, issue, rules) - return rules if user.admin? || !issue.confidential? - - unless issue.author == user || issue.assignee == user || issue.project.team.member?(user, Gitlab::Access::REPORTER) - rules.delete(:admin_issue) - rules.delete(:read_issue) - rules.delete(:update_issue) - end - - rules - end - - def project_group_member?(project, user) - project.group && - ( - project.group.members.exists?(user_id: user.id) || - project.group.requesters.exists?(user_id: user.id) - ) + def uncached_allowed(user, subject) + BasePolicy.class_for(subject).abilities(user, subject) end end end diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index b01a244032d..2340453831e 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -1,4 +1,8 @@ class AbuseReport < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :message, pipeline: :single_line + belongs_to :reporter, class_name: 'User' belongs_to :user @@ -7,6 +11,9 @@ class AbuseReport < ActiveRecord::Base validates :message, presence: true validates :user_id, uniqueness: { message: 'has already been reported' } + # For CacheMarkdownField + alias_method :author, :reporter + def remove_user(deleted_by:) user.block DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true) diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 4cf8dd9a8ce..e4106e1c2e9 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,4 +1,8 @@ class Appearance < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :description + validates :title, presence: true validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 8c19d9dc9c8..bb60cc8736c 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,5 +1,7 @@ class ApplicationSetting < ActiveRecord::Base + include CacheMarkdownField include TokenAuthenticatable + add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token @@ -16,6 +18,12 @@ class ApplicationSetting < ActiveRecord::Base serialize :disabled_oauth_sign_in_sources, Array serialize :domain_whitelist, Array serialize :domain_blacklist, Array + serialize :repository_storages + + cache_markdown_field :sign_in_text + cache_markdown_field :help_page_text + cache_markdown_field :shared_runners_text, pipeline: :plain_markdown + cache_markdown_field :after_sign_up_text attr_accessor :domain_whitelist_raw, :domain_blacklist_raw @@ -55,6 +63,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, if: :akismet_enabled + validates :koding_url, + presence: true, + if: :koding_enabled + validates :max_attachment_size, presence: true, numericality: { only_integer: true, greater_than: 0 } @@ -63,9 +75,8 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: 0 } - validates :repository_storage, - presence: true, - inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + validates :repository_storages, presence: true + validate :check_repository_storages validates :enabled_git_access_protocol, inclusion: { in: %w(ssh http), allow_blank: true, allow_nil: true } @@ -74,6 +85,18 @@ class ApplicationSetting < ActiveRecord::Base presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' }, if: :domain_blacklist_enabled? + validates :housekeeping_incremental_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: 0 } + + validates :housekeeping_full_repack_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_incremental_repack_period } + + validates :housekeeping_gc_period, + presence: true, + numericality: { only_integer: true, greater_than: :housekeeping_full_repack_period } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -142,19 +165,26 @@ class ApplicationSetting < ActiveRecord::Base default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], + import_sources: Gitlab::ImportSources.values, shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, two_factor_grace_period: 48, recaptcha_enabled: false, akismet_enabled: false, + koding_enabled: false, + koding_url: nil, repository_checks_enabled: true, disabled_oauth_sign_in_sources: [], send_user_confirmation_email: false, container_registry_token_expire_delay: 5, - repository_storage: 'default', + repository_storages: ['default'], user_default_external: false, + housekeeping_enabled: true, + housekeeping_bitmaps_enabled: true, + housekeeping_incremental_repack_period: 10, + housekeeping_full_repack_period: 50, + housekeeping_gc_period: 200, ) end @@ -188,6 +218,25 @@ class ApplicationSetting < ActiveRecord::Base self.domain_blacklist_raw = file.read end + def repository_storages + Array(read_attribute(:repository_storages)) + end + + # repository_storage is still required in the API. Remove in 9.0 + def repository_storage + repository_storages.first + end + + def repository_storage=(value) + self.repository_storages = [value] + end + + # Choose one of the available repository storage options. Currently all have + # equal weighting. + def pick_repository_storage + repository_storages.sample + end + def runners_registration_token ensure_runners_registration_token! end @@ -195,4 +244,12 @@ class ApplicationSetting < ActiveRecord::Base def health_check_access_token ensure_health_check_access_token! end + + private + + def check_repository_storages + invalid = repository_storages - Gitlab.config.repositories.storages.keys + errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless + invalid.empty? + end end diff --git a/app/models/blob.rb b/app/models/blob.rb index 12cc5aaafba..ab92e820335 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -22,6 +22,18 @@ class Blob < SimpleDelegator new(blob) end + # Returns the data of the blob. + # + # If the blob is a text based blob the content is converted to UTF-8 and any + # invalid byte sequences are replaced. + def data + if binary? + super + else + @data ||= super.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) + end + end + def no_highlighting? size && size > 1.megabyte end diff --git a/app/models/board.rb b/app/models/board.rb new file mode 100644 index 00000000000..c56422914a9 --- /dev/null +++ b/app/models/board.rb @@ -0,0 +1,15 @@ +class Board < ActiveRecord::Base + belongs_to :project + + has_many :lists, -> { order(:list_type, :position) }, dependent: :delete_all + + validates :project, presence: true + + def backlog_list + lists.merge(List.backlog).take + end + + def done_list + lists.merge(List.done).take + end +end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 61498140f27..cb40f33932a 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -1,6 +1,9 @@ class BroadcastMessage < ActiveRecord::Base + include CacheMarkdownField include Sortable + cache_markdown_field :message, pipeline: :broadcast_message + validates :message, presence: true validates :starts_at, presence: true validates :ends_at, presence: true diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4c84f4c21c5..bf5f92f8462 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,7 +1,10 @@ module Ci class Build < CommitStatus - belongs_to :runner, class_name: 'Ci::Runner' - belongs_to :trigger_request, class_name: 'Ci::TriggerRequest' + include TokenAuthenticatable + include AfterCommitQueue + + belongs_to :runner + belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' serialize :options @@ -23,7 +26,10 @@ module Ci acts_as_taggable + add_authentication_token_field :token + before_save :update_artifacts_size, if: :artifacts_file_changed? + before_save :ensure_token before_destroy { project } after_create :execute_hooks @@ -38,6 +44,7 @@ module Ci new_build.status = 'pending' new_build.runner_id = nil new_build.trigger_request_id = nil + new_build.token = nil new_build.save end @@ -62,28 +69,27 @@ module Ci status_event: 'enqueue' ) MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build) + build.pipeline.mark_as_processable_after_stage(build.stage_idx) new_build end end state_machine :status do after_transition pending: :running do |build| - build.execute_hooks + build.run_after_commit do + BuildHooksWorker.perform_async(id) + end end after_transition any => [:success, :failed, :canceled] do |build| - build.update_coverage - build.execute_hooks + build.run_after_commit do + BuildFinishedWorker.perform_async(id) + end end after_transition any => [:success] do |build| - if build.environment.present? - service = CreateDeploymentService.new(build.project, build.user, - environment: build.environment, - sha: build.sha, - ref: build.ref, - tag: build.tag) - service.execute(build) + build.run_after_commit do + BuildSuccessWorker.perform_async(id) end end end @@ -97,7 +103,7 @@ module Ci end def playable? - project.builds_enabled? && commands.present? && manual? + project.builds_enabled? && commands.present? && manual? && skipped? end def play(current_user = nil) @@ -127,13 +133,17 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def trace_html - trace_with_state[:html] || '' + def trace_html(**args) + trace_with_state(**args)[:html] || '' end - def trace_with_state(state = nil) - trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present? - trace_with_state || {} + def trace_with_state(state: nil, last_lines: nil) + trace_ansi = trace(last_lines: last_lines) + if trace_ansi.present? + Ci::Ansi2html.convert(trace_ansi, state) + else + {} + end end def timeout @@ -147,6 +157,7 @@ module Ci variables += runner.predefined_variables if runner variables += project.container_registry_variables variables += yaml_variables + variables += user_variables variables += project.secret_variables variables += trigger_request.user_variables if trigger_request variables @@ -171,7 +182,7 @@ module Ci end def repo_url - auth = "gitlab-ci-token:#{token}@" + auth = "gitlab-ci-token:#{ensure_token!}@" project.http_url_to_repo.sub(/^https?:\/\//) do |prefix| prefix + auth end @@ -207,29 +218,34 @@ module Ci end end + def has_trace_file? + File.exist?(path_to_trace) || has_old_trace_file? + end + def has_trace? raw_trace.present? end - def raw_trace - if File.file?(path_to_trace) - File.read(path_to_trace) - elsif project.ci_id && File.file?(old_path_to_trace) - # Temporary fix for build trace data integrity - File.read(old_path_to_trace) + def raw_trace(last_lines: nil) + if File.exist?(trace_file_path) + Gitlab::Ci::TraceReader.new(trace_file_path). + read(last_lines: last_lines) else # backward compatibility read_attribute :trace end end - def trace - trace = raw_trace - if project && trace.present? && project.runners_token.present? - trace.gsub(project.runners_token, 'xxxxxx') - else - trace - end + ## + # Deprecated + # + # This is a hotfix for CI build data integrity, see #4246 + def has_old_trace_file? + project.ci_id && File.exist?(old_path_to_trace) + end + + def trace(last_lines: nil) + hide_secrets(raw_trace(last_lines: last_lines)) end def trace_length @@ -242,6 +258,7 @@ module Ci def trace=(trace) recreate_trace_dir + trace = hide_secrets(trace) File.write(path_to_trace, trace) end @@ -255,12 +272,22 @@ module Ci def append_trace(trace_part, offset) recreate_trace_dir + trace_part = hide_secrets(trace_part) + File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) File.open(path_to_trace, 'ab') do |f| f.write(trace_part) end end + def trace_file_path + if has_old_trace_file? + old_path_to_trace + else + path_to_trace + end + end + def dir_to_trace File.join( Settings.gitlab_ci.builds_path, @@ -322,12 +349,8 @@ module Ci ) end - def token - project.runners_token - end - def valid_token?(token) - project.valid_runners_token?(token) + self.token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) end def has_tags? @@ -416,6 +439,15 @@ module Ci read_attribute(:yaml_variables) || build_attributes_from_config[:yaml_variables] || [] end + def user_variables + return [] if user.blank? + + [ + { key: 'GITLAB_USER_ID', value: user.id.to_s, public: true }, + { key: 'GITLAB_USER_EMAIL', value: user.email, public: true } + ] + end + private def update_artifacts_size @@ -451,6 +483,7 @@ module Ci ] variables << { key: 'CI_BUILD_TAG', value: ref, public: true } if tag? variables << { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true } if trigger_request + variables << { key: 'CI_BUILD_MANUAL', value: 'true', public: true } if manual? variables end @@ -459,5 +492,14 @@ module Ci pipeline.config_processor.build_attributes(name) end + + def hide_secrets(trace) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) + trace + end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 130afeb724e..3fee6c18770 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -1,23 +1,25 @@ module Ci class Pipeline < ActiveRecord::Base extend Ci::Model - include Statuseable + include HasStatus + include Importable + include AfterCommitQueue self.table_name = 'ci_commits' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :project, foreign_key: :gl_project_id belongs_to :user has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id - has_many :builds, class_name: 'Ci::Build', foreign_key: :commit_id - has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest', foreign_key: :commit_id + has_many :builds, foreign_key: :commit_id + has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id - validates_presence_of :sha - validates_presence_of :ref - validates_presence_of :status - validate :valid_commit_sha + validates_presence_of :sha, unless: :importing? + validates_presence_of :ref, unless: :importing? + validates_presence_of :status, unless: :importing? + validate :valid_commit_sha, unless: :importing? - after_save :keep_around_commits + after_create :keep_around_commits, unless: :importing? delegate :stages, to: :statuses @@ -28,45 +30,68 @@ module Ci end event :run do - transition any => :running + transition any - [:running] => :running end event :skip do - transition any => :skipped + transition any - [:skipped] => :skipped end event :drop do - transition any => :failed + transition any - [:failed] => :failed end event :succeed do - transition any => :success + transition any - [:success] => :success end event :cancel do - transition any => :canceled + transition any - [:canceled] => :canceled end + # IMPORTANT + # Do not add any operations to this state_machine + # Create a separate worker for each new operation + before_transition [:created, :pending] => :running do |pipeline| pipeline.started_at = Time.now end before_transition any => [:success, :failed, :canceled] do |pipeline| pipeline.finished_at = Time.now + pipeline.update_duration end - before_transition do |pipeline| - pipeline.update_duration + after_transition [:created, :pending] => :running do |pipeline| + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + end + + after_transition any => [:success] do |pipeline| + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + end + + after_transition [:created, :pending, :running] => :success do |pipeline| + pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) } end after_transition do |pipeline, transition| - pipeline.execute_hooks unless transition.loopback? + next if transition.loopback? + + pipeline.run_after_commit do + PipelineHooksWorker.perform_async(id) + end + end + + after_transition any => [:success, :failed] do |pipeline| + pipeline.run_after_commit do + PipelineNotificationWorker.perform_async(pipeline.id) + end end end # ref can't be HEAD or SHA, can only be branch/tag name - scope :latest_successful_for, ->(ref = default_branch) do - where(ref: ref).success.order(id: :desc).limit(1) + def self.latest_successful_for(ref) + where(ref: ref).order(id: :desc).success.first end def self.truncate_sha(sha) @@ -78,10 +103,23 @@ module Ci CommitStatus.where(pipeline: pluck(:id)).stages end + def self.total_duration + where.not(duration: nil).sum(:duration) + end + + def stages_with_latest_statuses + statuses.latest.includes(project: :namespace).order(:stage_idx).group_by(&:stage) + end + def project_id project.id end + # For now the only user who participates is the user who triggered + def participants(_current_user = nil) + Array(user) + end + def valid_commit_sha if self.sha == Gitlab::Git::BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") @@ -124,7 +162,7 @@ module Ci def retryable? builds.latest.any? do |build| - build.failed? && build.retryable? + (build.failed? || build.canceled?) && build.retryable? end end @@ -142,6 +180,10 @@ module Ci end end + def mark_as_processable_after_stage(stage_idx) + builds.skipped.where('stage_idx > ?', stage_idx).find_each(&:process) + end + def latest? return false unless ref commit = project.commit(ref) @@ -173,7 +215,7 @@ module Ci end def has_warnings? - builds.latest.ignored.any? + builds.latest.failed_but_allowed.any? end def config_processor @@ -228,14 +270,16 @@ module Ci Ci::ProcessPipelineService.new(project, user).execute(self) end - def build_updated - case latest_builds_status - when 'pending' then enqueue - when 'running' then run - when 'success' then succeed - when 'failed' then drop - when 'canceled' then cancel - when 'skipped' then skip + def update_status + Gitlab::OptimisticLocking.retry_lock(self) do + case latest_builds_status + when 'pending' then enqueue + when 'running' then run + when 'success' then succeed + when 'failed' then drop + when 'canceled' then cancel + when 'skipped' then skip + end end end @@ -245,8 +289,17 @@ module Ci ] end + def queued_duration + return unless started_at + + seconds = (started_at - created_at).to_i + seconds unless seconds.zero? + end + def update_duration - self.duration = statuses.latest.duration + return unless started_at + + self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self) end def execute_hooks @@ -255,6 +308,14 @@ module Ci project.execute_services(data, :pipeline_hooks) end + # Merge requests for which the current pipeline is running against + # the merge request's latest commit. + def merge_requests + @merge_requests ||= project.merge_requests + .where(source_branch: self.ref) + .select { |merge_request| merge_request.pipeline.try(:id) == self.id } + end + private def pipeline_data diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 49f05f881a2..123930273e0 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -2,13 +2,13 @@ module Ci class Runner < ActiveRecord::Base extend Ci::Model - LAST_CONTACT_TIME = 5.minutes.ago + LAST_CONTACT_TIME = 1.hour.ago AVAILABLE_SCOPES = %w[specific shared active paused online] FORM_EDITABLE = %i[description tag_list active run_untagged locked] - has_many :builds, class_name: 'Ci::Build' - has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject' - has_many :projects, through: :runner_projects, class_name: '::Project', foreign_key: :gl_project_id + has_many :builds + has_many :runner_projects, dependent: :destroy + has_many :projects, through: :runner_projects, foreign_key: :gl_project_id has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 4b44ffa886e..1f9baeca5b1 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -2,8 +2,8 @@ module Ci class RunnerProject < ActiveRecord::Base extend Ci::Model - belongs_to :runner, class_name: 'Ci::Runner' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :runner + belongs_to :project, foreign_key: :gl_project_id validates_uniqueness_of :runner_id, scope: :gl_project_id end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index a0b19b51a12..62889fe80d8 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -4,8 +4,8 @@ module Ci acts_as_paranoid - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id - has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest' + belongs_to :project, foreign_key: :gl_project_id + has_many :trigger_requests, dependent: :destroy validates_presence_of :token validates_uniqueness_of :token diff --git a/app/models/ci/trigger_request.rb b/app/models/ci/trigger_request.rb index fc674871743..2b807731d0d 100644 --- a/app/models/ci/trigger_request.rb +++ b/app/models/ci/trigger_request.rb @@ -2,9 +2,9 @@ module Ci class TriggerRequest < ActiveRecord::Base extend Ci::Model - belongs_to :trigger, class_name: 'Ci::Trigger' - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id - has_many :builds, class_name: 'Ci::Build' + belongs_to :trigger + belongs_to :pipeline, foreign_key: :commit_id + has_many :builds serialize :variables diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index c9c47ec7419..94d9e2b3208 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -1,8 +1,8 @@ module Ci class Variable < ActiveRecord::Base extend Ci::Model - - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + + belongs_to :project, foreign_key: :gl_project_id validates_uniqueness_of :key, scope: :gl_project_id validates :key, @@ -11,7 +11,9 @@ module Ci format: { with: /\A[a-zA-Z0-9_]+\z/, message: "can contain only letters, digits and '_'." } - attr_encrypted :value, + scope :order_key_asc, -> { reorder(key: :asc) } + + attr_encrypted :value, mode: :per_attribute_iv_and_salt, insecure_mode: true, key: Gitlab::Application.secrets.db_key_base, diff --git a/app/models/commit.rb b/app/models/commit.rb index cc413448ce8..9e7fde9503d 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -108,15 +108,6 @@ class Commit @diff_line_count end - # Returns a string describing the commit for use in a link title - # - # Example - # - # "Commit: Alex Denisov - Project git clone panel" - def link_title - "Commit: #{author_name} - #{title}" - end - # Returns the commits title. # # Usually, the commit title is the first line of the commit message. @@ -229,18 +220,25 @@ class Commit def diff_refs Gitlab::Diff::DiffRefs.new( - base_sha: self.parent_id || self.sha, + base_sha: self.parent_id || Gitlab::Git::BLANK_SHA, head_sha: self.sha ) end def pipelines - @pipeline ||= project.pipelines.where(sha: sha) + project.pipelines.where(sha: sha) end - def status - return @status if defined?(@status) - @status ||= pipelines.status + def status(ref = nil) + @statuses ||= {} + + if @statuses.key?(ref) + @statuses[ref] + elsif ref + @statuses[ref] = pipelines.where(ref: ref).status + else + @statuses[ref] = pipelines.status + end end def revert_branch_name diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 630ee9601e0..ac2477fd973 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -4,12 +4,10 @@ # # range = CommitRange.new('f3f85602...e86e1013', project) # range.exclude_start? # => false -# range.reference_title # => "Commits f3f85602 through e86e1013" # range.to_s # => "f3f85602...e86e1013" # # range = CommitRange.new('f3f856029bc5f966c5a7ee24cf7efefdd20e6019..e86e1013709735be5bb767e2b228930c543f25ae', project) # range.exclude_start? # => true -# range.reference_title # => "Commits f3f85602^ through e86e1013" # range.to_param # => {from: "f3f856029bc5f966c5a7ee24cf7efefdd20e6019^", to: "e86e1013709735be5bb767e2b228930c543f25ae"} # range.to_s # => "f3f85602..e86e1013" # @@ -82,7 +80,7 @@ class CommitRange end def inspect - %(#<#{self.class}:#{object_id} #{to_s}>) + %(#<#{self.class}:#{object_id} #{self}>) end def to_s @@ -109,11 +107,6 @@ class CommitRange reference end - # Returns a String for use in a link's title attribute - def reference_title - "Commits #{sha_start} through #{sha_to}" - end - # Return a Hash of parameters for passing to a URL helper # # See `namespace_project_compare_url` diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 703ca90edb6..d159fc6c5c7 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -1,10 +1,11 @@ class CommitStatus < ActiveRecord::Base - include Statuseable + include HasStatus include Importable + include AfterCommitQueue self.table_name = 'ci_builds' - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + belongs_to :project, foreign_key: :gl_project_id belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id belongs_to :user @@ -21,15 +22,37 @@ class CommitStatus < ActiveRecord::Base where(id: max_id.group(:name, :commit_id)) end + scope :retried, -> { where.not(id: latest) } scope :ordered, -> { order(:name) } - scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + quoted_when = connection.quote_column_name('when') + # We want to ignore failed_but_allowed jobs + where("allow_failure = ? OR status IN (?)", + false, all_state_names - [:failed, :canceled]). + # We want to ignore skipped manual jobs + where("#{quoted_when} <> ? OR status <> ?", 'manual', 'skipped'). + # We want to ignore skipped on_failure + where("#{quoted_when} <> ? OR status <> ?", 'on_failure', 'skipped') + end + + scope :latest_ci_stages, -> { latest.ordered.includes(project: :namespace) } + scope :retried_ci_stages, -> { retried.ordered.includes(project: :namespace) } state_machine :status do event :enqueue do transition [:created, :skipped] => :pending end + event :process do + transition skipped: :created + end + event :run do transition pending: :running end @@ -50,35 +73,37 @@ class CommitStatus < ActiveRecord::Base transition [:created, :pending, :running] => :canceled end - after_transition created: [:pending, :running] do |commit_status| - commit_status.update_attributes queued_at: Time.now + before_transition created: [:pending, :running] do |commit_status| + commit_status.queued_at = Time.now end - after_transition [:created, :pending] => :running do |commit_status| - commit_status.update_attributes started_at: Time.now + before_transition [:created, :pending] => :running do |commit_status| + commit_status.started_at = Time.now end - after_transition any => [:success, :failed, :canceled] do |commit_status| - commit_status.update_attributes finished_at: Time.now - end - - # We use around_transition to process pipeline on next stages as soon as possible, before the `after_*` is executed - around_transition any => [:success, :failed, :canceled] do |commit_status, block| - block.call - - commit_status.pipeline.try(:process!) + before_transition any => [:success, :failed, :canceled] do |commit_status| + commit_status.finished_at = Time.now end after_transition do |commit_status, transition| - commit_status.pipeline.try(:build_updated) unless transition.loopback? - end - - after_transition [:created, :pending, :running] => :success do |commit_status| - MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.pipeline.project, nil).trigger(commit_status) + next if transition.loopback? + + commit_status.run_after_commit do + pipeline.try do |pipeline| + if complete? + PipelineProcessWorker.perform_async(pipeline.id) + else + PipelineUpdateWorker.perform_async(pipeline.id) + end + end + end end after_transition any => :failed do |commit_status| - MergeRequests::AddTodoWhenBuildFailsService.new(commit_status.pipeline.project, nil).execute(commit_status) + commit_status.run_after_commit do + MergeRequests::AddTodoWhenBuildFailsService + .new(pipeline.project, nil).execute(self) + end end end @@ -88,6 +113,10 @@ class CommitStatus < ActiveRecord::Base pipeline.before_sha || Gitlab::Git::BLANK_SHA end + def group_name + name.gsub(/\d+[\s:\/\\]+\d+\s*/, '').strip + end + def self.stages # We group by stage name, but order stages by theirs' index unscoped.from(all, :sg).group('stage').order('max(stage_idx)', 'stage').pluck('sg.stage') @@ -102,18 +131,16 @@ class CommitStatus < ActiveRecord::Base end end - def ignored? + def failed_but_allowed? allow_failure? && (failed? || canceled?) end + def playable? + false + end + def duration - duration = - if started_at && finished_at - finished_at - started_at - elsif started_at - Time.now - started_at - end - duration + calculate_duration end def stuck? diff --git a/app/models/compare.rb b/app/models/compare.rb index 4856510f526..3a8bbcb1acd 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -11,9 +11,10 @@ class Compare end end - def initialize(compare, project) + def initialize(compare, project, straight: false) @compare = compare @project = project + @straight = straight end def commits @@ -45,6 +46,18 @@ class Compare end end + def start_commit_sha + start_commit.try(:sha) + end + + def base_commit_sha + base_commit.try(:sha) + end + + def head_commit_sha + commit.try(:sha) + end + def raw_diffs(*args) @compare.diffs(*args) end @@ -58,9 +71,9 @@ class Compare def diff_refs Gitlab::Diff::DiffRefs.new( - base_sha: base_commit.try(:sha), - start_sha: start_commit.try(:sha), - head_sha: commit.try(:sha) + base_sha: @straight ? start_commit_sha : base_commit_sha, + start_sha: start_commit_sha, + head_sha: head_commit_sha ) end end diff --git a/app/models/concerns/access_requestable.rb b/app/models/concerns/access_requestable.rb index eedd32a729f..62bc6b809f4 100644 --- a/app/models/concerns/access_requestable.rb +++ b/app/models/concerns/access_requestable.rb @@ -8,9 +8,6 @@ module AccessRequestable extend ActiveSupport::Concern def request_access(user) - members.create( - access_level: Gitlab::Access::DEVELOPER, - user: user, - requested_at: Time.now.utc) + Members::RequestAccessService.new(self, user).execute end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 800a16ab246..073ac4c1b65 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -2,7 +2,7 @@ module Awardable extend ActiveSupport::Concern included do - has_many :award_emoji, -> { includes(:user) }, as: :awardable, dependent: :destroy + has_many :award_emoji, -> { includes(:user).order(:id) }, as: :awardable, dependent: :destroy if self < Participable # By default we always load award_emoji user association @@ -59,6 +59,24 @@ module Awardable true end + def awardable_votes?(name) + AwardEmoji::UPVOTE_NAME == name || AwardEmoji::DOWNVOTE_NAME == name + end + + def user_can_award?(current_user, name) + if user_authored?(current_user) + !awardable_votes?(normalize_name(name)) + else + true + end + end + + def user_authored?(current_user) + author = self.respond_to?(:author) ? self.author : self.user + + author == current_user + end + def awarded_emoji?(emoji_name, current_user) award_emoji.where(name: emoji_name, user: current_user).exists? end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb new file mode 100644 index 00000000000..90bd6490a02 --- /dev/null +++ b/app/models/concerns/cache_markdown_field.rb @@ -0,0 +1,131 @@ +# This module takes care of updating cache columns for Markdown-containing +# fields. Use like this in the body of your class: +# +# include CacheMarkdownField +# cache_markdown_field :foo +# cache_markdown_field :bar +# cache_markdown_field :baz, pipeline: :single_line +# +# Corresponding foo_html, bar_html and baz_html fields should exist. +module CacheMarkdownField + # Knows about the relationship between markdown and html field names, and + # stores the rendering contexts for the latter + class FieldData + extend Forwardable + + def initialize + @data = {} + end + + def_delegators :@data, :[], :[]= + def_delegator :@data, :keys, :markdown_fields + + def html_field(markdown_field) + "#{markdown_field}_html" + end + + def html_fields + markdown_fields.map {|field| html_field(field) } + end + end + + # Dynamic registries don't really work in Rails as it's not guaranteed that + # every class will be loaded, so hardcode the list. + CACHING_CLASSES = %w[ + AbuseReport + Appearance + ApplicationSetting + BroadcastMessage + Issue + Label + MergeRequest + Milestone + Namespace + Note + Project + Release + Snippet + ] + + def self.caching_classes + CACHING_CLASSES.map(&:constantize) + end + + extend ActiveSupport::Concern + + included do + cattr_reader :cached_markdown_fields do + FieldData.new + end + + # Returns the default Banzai render context for the cached markdown field. + def banzai_render_context(field) + raise ArgumentError.new("Unknown field: #{field.inspect}") unless + cached_markdown_fields.markdown_fields.include?(field) + + # Always include a project key, or Banzai complains + project = self.project if self.respond_to?(:project) + context = cached_markdown_fields[field].merge(project: project) + + # Banzai is less strict about authors, so don't always have an author key + context[:author] = self.author if self.respond_to?(:author) + + context + end + + # Allow callers to look up the cache field name, rather than hardcoding it + def markdown_cache_field_for(field) + raise ArgumentError.new("Unknown field: #{field}") unless + cached_markdown_fields.markdown_fields.include?(field) + + cached_markdown_fields.html_field(field) + end + + # Always exclude _html fields from attributes (including serialization). + # They contain unredacted HTML, which would be a security issue + alias_method :attributes_before_markdown_cache, :attributes + def attributes + attrs = attributes_before_markdown_cache + + cached_markdown_fields.html_fields.each do |field| + attrs.delete(field) + end + + attrs + end + end + + class_methods do + private + + # Specify that a field is markdown. Its rendered output will be cached in + # a corresponding _html field. Any custom rendering options may be provided + # as a context. + def cache_markdown_field(markdown_field, context = {}) + raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless + CacheMarkdownField::CACHING_CLASSES.include?(self.to_s) + + cached_markdown_fields[markdown_field] = context + + html_field = cached_markdown_fields.html_field(markdown_field) + cache_method = "#{markdown_field}_cache_refresh".to_sym + invalidation_method = "#{html_field}_invalidated?".to_sym + + define_method(cache_method) do + html = Banzai::Renderer.cacheless_render_field(self, markdown_field) + __send__("#{html_field}=", html) + true + end + + # The HTML becomes invalid if any dependent fields change. For now, assume + # author and project invalidate the cache in all circumstances. + define_method(invalidation_method) do + changed_fields = changed_attributes.keys + invalidations = changed_fields & [markdown_field.to_s, "author", "project"] + !invalidations.empty? + end + + before_save cache_method, if: invalidation_method + end + end +end diff --git a/app/models/concerns/expirable.rb b/app/models/concerns/expirable.rb new file mode 100644 index 00000000000..b66ba08dc59 --- /dev/null +++ b/app/models/concerns/expirable.rb @@ -0,0 +1,19 @@ +module Expirable + extend ActiveSupport::Concern + + included do + scope :expired, -> { where('expires_at <= ?', Time.current) } + end + + def expired? + expires? && expires_at <= Time.current + end + + def expires? + expires_at.present? + end + + def expires_soon? + expires? && expires_at < 7.days.from_now + end +end diff --git a/app/models/concerns/statuseable.rb b/app/models/concerns/has_status.rb index 5d4b0a86899..ef3e73a4072 100644 --- a/app/models/concerns/statuseable.rb +++ b/app/models/concerns/has_status.rb @@ -1,43 +1,40 @@ -module Statuseable +module HasStatus extend ActiveSupport::Concern AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped] STARTED_STATUSES = %w[running success failed skipped] ACTIVE_STATUSES = %w[pending running] COMPLETED_STATUSES = %w[success failed canceled] + ORDERED_STATUSES = %w[failed pending running canceled success skipped] class_methods do def status_sql - scope = all.relevant + scope = if respond_to?(:exclude_ignored) + exclude_ignored + else + all + end builds = scope.select('count(*)').to_sql + created = scope.created.select('count(*)').to_sql success = scope.success.select('count(*)').to_sql - ignored = scope.ignored.select('count(*)').to_sql if scope.respond_to?(:ignored) - ignored ||= '0' pending = scope.pending.select('count(*)').to_sql running = scope.running.select('count(*)').to_sql - canceled = scope.canceled.select('count(*)').to_sql skipped = scope.skipped.select('count(*)').to_sql + canceled = scope.canceled.select('count(*)').to_sql - deduce_status = "(CASE - WHEN (#{builds})=0 THEN NULL - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success})+(#{ignored})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{pending})+(#{skipped}) THEN 'pending' - WHEN (#{builds})=(#{canceled})+(#{success})+(#{ignored})+(#{skipped}) THEN 'canceled' - WHEN (#{running})+(#{pending})>0 THEN 'running' + "(CASE + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})+(#{created})>0 THEN 'running' ELSE 'failed' END)" - - deduce_status end def status - all.pluck(self.status_sql).first - end - - def duration - duration_array = all.map(&:duration).compact - duration_array.reduce(:+) + all.pluck(status_sql).first end def started_at @@ -47,6 +44,10 @@ module Statuseable def finished_at all.maximum(:finished_at) end + + def all_state_names + state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } + end end included do @@ -85,4 +86,14 @@ module Statuseable def complete? COMPLETED_STATUSES.include?(status) end + + private + + def calculate_duration + if started_at && finished_at + finished_at - started_at + elsif started_at + Time.now - started_at + end + end end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index cbae1cd439b..664bb594aa9 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -6,13 +6,18 @@ # module Issuable extend ActiveSupport::Concern + include CacheMarkdownField include Participable include Mentionable include Subscribable include StripAttribute include Awardable + include Taskable included do + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description + belongs_to :author, class_name: "User" belongs_to :assignee, class_name: "User" belongs_to :updated_by, class_name: "User" @@ -28,10 +33,13 @@ module Issuable loaded? && to_a.all? { |note| note.association(:award_emoji).loaded? } end end + has_many :label_links, as: :target, dependent: :destroy has_many :labels, through: :label_links has_many :todos, as: :target, dependent: :destroy + has_one :metrics + validates :author, presence: true validates :title, presence: true, length: { within: 0..255 } @@ -81,12 +89,19 @@ module Issuable acts_as_paranoid after_save :update_assignee_cache_counts, if: :assignee_id_changed? + after_save :record_metrics def update_assignee_cache_counts # make sure we flush the cache for both the old *and* new assignee User.find(assignee_id_was).update_cache_counts if assignee_id_was assignee.update_cache_counts if assignee end + + # We want to use optimistic lock for cases when only title or description are involved + # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html + def locking_enabled? + title_changed? || description_changed? + end end module ClassMethods @@ -131,7 +146,16 @@ module Issuable end def order_labels_priority(excluded_labels: []) - select("#{table_name}.*, (#{highest_label_priority(excluded_labels).to_sql}) AS highest_priority"). + params = { + target_type: name, + target_column: "#{table_name}.id", + project_column: "#{table_name}.#{project_foreign_key}", + excluded_labels: excluded_labels + } + + highest_priority = highest_label_priority(params).to_sql + + select("#{table_name}.*, (#{highest_priority}) AS highest_priority"). group(arel_table[:id]). reorder(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')) end @@ -160,18 +184,8 @@ module Issuable grouping_columns end - private - - def highest_label_priority(excluded_labels) - query = Label.select(Label.arel_table[:priority].minimum). - joins(:label_links). - where(label_links: { target_type: name }). - where("label_links.target_id = #{table_name}.id"). - reorder(nil) - - query.where.not(title: excluded_labels) if excluded_labels.present? - - query + def to_ability_name + model_name.singular end end @@ -227,18 +241,6 @@ module Issuable labels.order('title ASC').pluck(:title) end - def remove_labels - labels.delete_all - end - - def add_labels_by_names(label_names) - label_names.each do |label_name| - label = project.labels.create_with(color: Label::DEFAULT_COLOR). - find_or_create_by(title: label_name.strip) - self.labels << label - end - end - # Convert this Issuable class name to a format usable by Ability definitions # # Examples: @@ -246,7 +248,7 @@ module Issuable # issuable.class # => MergeRequest # issuable.to_ability_name # => "merge_request" def to_ability_name - self.class.to_s.underscore + self.class.to_ability_name end # Returns a Hash of attributes to be used for Twitter card metadata @@ -287,4 +289,14 @@ module Issuable def can_move?(*) false end + + def assignee_or_author?(user) + # We're comparing IDs here so we don't need to load any associations. + author_id == user.id || assignee_id == user.id + end + + def record_metrics + metrics = self.metrics || create_metrics + metrics.record! + end end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index ec9e0f1b1d0..eb2ff0428f6 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -43,19 +43,15 @@ module Mentionable self end - def all_references(current_user = nil, text = nil, extractor: nil) + def all_references(current_user = nil, extractor: nil) extractor ||= Gitlab::ReferenceExtractor. new(project, current_user) - if text - extractor.analyze(text, author: author) - else - self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) - options = options.merge(cache_key: [self, attr], author: author) + self.class.mentionable_attrs.each do |attr, options| + text = __send__(attr) + options = options.merge(cache_key: [self, attr], author: author) - extractor.analyze(text, options) - end + extractor.analyze(text, options) end extractor @@ -66,8 +62,8 @@ module Mentionable end # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. - def referenced_mentionables(current_user = self.author, text = nil) - refs = all_references(current_user, text) + def referenced_mentionables(current_user = self.author) + refs = all_references(current_user) refs = (refs.issues + refs.merge_requests + refs.commits) # We're using this method instead of Array diffing because that requires @@ -77,8 +73,8 @@ module Mentionable end # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+. - def create_cross_references!(author = self.author, without = [], text = nil) - refs = referenced_mentionables(author, text) + def create_cross_references!(author = self.author, without = []) + refs = referenced_mentionables(author) # We're using this method instead of Array diffing because that requires # both of the object's `hash` values to be the same, which may not be the @@ -97,10 +93,7 @@ module Mentionable return if changes.empty? - original_text = changes.collect { |_, vals| vals.first }.join(' ') - - preexisting = referenced_mentionables(author, original_text) - create_cross_references!(author, preexisting) + create_cross_references!(author) end private diff --git a/app/models/concerns/note_on_diff.rb b/app/models/concerns/note_on_diff.rb index 4be6a2f621b..b8dd27a7afe 100644 --- a/app/models/concerns/note_on_diff.rb +++ b/app/models/concerns/note_on_diff.rb @@ -17,6 +17,10 @@ module NoteOnDiff raise NotImplementedError end + def original_line_code + raise NotImplementedError + end + def diff_attributes raise NotImplementedError end @@ -24,4 +28,8 @@ module NoteOnDiff def can_be_award_emoji? false end + + def to_discussion + Discussion.new([self]) + end end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb new file mode 100644 index 00000000000..6d88951c713 --- /dev/null +++ b/app/models/concerns/project_features_compatibility.rb @@ -0,0 +1,37 @@ +# Makes api V3 compatible with old project features permissions methods +# +# After migrating issues_enabled merge_requests_enabled builds_enabled snippets_enabled and wiki_enabled +# fields to a new table "project_features", support for the old fields is still needed in the API. + +module ProjectFeaturesCompatibility + extend ActiveSupport::Concern + + def wiki_enabled=(value) + write_feature_attribute(:wiki_access_level, value) + end + + def builds_enabled=(value) + write_feature_attribute(:builds_access_level, value) + end + + def merge_requests_enabled=(value) + write_feature_attribute(:merge_requests_access_level, value) + end + + def issues_enabled=(value) + write_feature_attribute(:issues_access_level, value) + end + + def snippets_enabled=(value) + write_feature_attribute(:snippets_access_level, value) + end + + private + + def write_feature_attribute(field, value) + build_project_feature unless project_feature + + access_level = Gitlab::Utils.to_boolean(value) ? ProjectFeature::ENABLED : ProjectFeature::DISABLED + project_feature.update_attribute(field, access_level) + end +end diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index 5a7b36070e7..7fd0905ee81 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -1,6 +1,11 @@ module ProtectedBranchAccess extend ActiveSupport::Concern + included do + scope :master, -> { where(access_level: Gitlab::Access::MASTER) } + scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } + end + def humanize self.class.human_access_levels[self.access_level] end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index 8b47b9e0abd..7edb0acd56c 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -35,5 +35,26 @@ module Sortable all end end + + private + + def highest_label_priority(target_type_column: nil, target_type: nil, target_column:, project_column:, excluded_labels: []) + query = Label.select(LabelPriority.arel_table[:priority].minimum). + left_join_priorities. + joins(:label_links). + where("label_priorities.project_id = #{project_column}"). + where("label_links.target_id = #{target_column}"). + reorder(nil) + + if target_type_column + query = query.where("label_links.target_type = #{target_type_column}") + else + query = query.where(label_links: { target_type: target_type }) + end + + query = query.where.not(title: excluded_labels) if excluded_labels.present? + + query + end end end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index ce54fe5d3bf..1aa97debe42 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -23,7 +23,7 @@ module Spammable def submittable_as_spam? if user_agent_detail - user_agent_detail.submittable? + user_agent_detail.submittable? && current_application_settings.akismet_enabled else false end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index df2a9e3e84b..ebc75100a54 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -52,11 +52,23 @@ module Taskable end # Return a string that describes the current state of this Taskable's task - # list items, e.g. "20 tasks (12 completed, 8 remaining)" - def task_status + # list items, e.g. "12 of 20 tasks completed" + def task_status(short: false) return '' if description.blank? + prep, completed = if short + ['/', ''] + else + [' of ', ' completed'] + end + sum = tasks.summary - "#{sum.item_count} tasks (#{sum.complete_count} completed, #{sum.incomplete_count} remaining)" + "#{sum.complete_count}#{prep}#{sum.item_count} #{'task'.pluralize(sum.item_count)}#{completed}" + end + + # Return a short string that describes the current state of this Taskable's + # task list items -- for small screens + def task_status_short + task_status(short: true) end end diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 24c7b26d223..04d30f46210 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -4,17 +4,21 @@ module TokenAuthenticatable private def write_new_token(token_field) - new_token = generate_token(token_field) + new_token = generate_available_token(token_field) write_attribute(token_field, new_token) end - def generate_token(token_field) + def generate_available_token(token_field) loop do - token = Devise.friendly_token + token = generate_token(token_field) break token unless self.class.unscoped.find_by(token_field => token) end end + def generate_token(token_field) + Devise.friendly_token + end + class_methods do def authentication_token_fields @token_fields || [] diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb new file mode 100644 index 00000000000..8ed4a56b19b --- /dev/null +++ b/app/models/cycle_analytics.rb @@ -0,0 +1,103 @@ +class CycleAnalytics + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + DEPLOYMENT_METRIC_STAGES = %i[production staging] + + def initialize(project, from:) + @project = project + @from = from + end + + def summary + @summary ||= Summary.new(@project, from: @from) + end + + def issue + calculate_metric(:issue, + Issue.arel_table[:created_at], + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]]) + end + + def plan + calculate_metric(:plan, + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]], + Issue::Metrics.arel_table[:first_mentioned_in_commit_at]) + end + + def code + calculate_metric(:code, + Issue::Metrics.arel_table[:first_mentioned_in_commit_at], + MergeRequest.arel_table[:created_at]) + end + + def test + calculate_metric(:test, + MergeRequest::Metrics.arel_table[:latest_build_started_at], + MergeRequest::Metrics.arel_table[:latest_build_finished_at]) + end + + def review + calculate_metric(:review, + MergeRequest.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:merged_at]) + end + + def staging + calculate_metric(:staging, + MergeRequest::Metrics.arel_table[:merged_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end + + def production + calculate_metric(:production, + Issue.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end + + private + + def calculate_metric(name, start_time_attrs, end_time_attrs) + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query_for(name), end_time_attrs, start_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + # Join table with a row for every <issue,merge_request> pair (where the merge request + # closes the given issue) with issue and merge request metrics included. The metrics + # are loaded with an inner join, so issues / merge requests without metrics are + # automatically excluded. + def base_query_for(name) + arel_table = MergeRequestsClosingIssues.arel_table + + # Load issues + query = arel_table.join(Issue.arel_table).on(Issue.arel_table[:id].eq(arel_table[:issue_id])). + join(Issue::Metrics.arel_table).on(Issue.arel_table[:id].eq(Issue::Metrics.arel_table[:issue_id])). + where(Issue.arel_table[:project_id].eq(@project.id)). + where(Issue.arel_table[:deleted_at].eq(nil)). + where(Issue.arel_table[:created_at].gteq(@from)) + + # Load merge_requests + query = query.join(MergeRequest.arel_table, Arel::Nodes::OuterJoin). + on(MergeRequest.arel_table[:id].eq(arel_table[:merge_request_id])). + join(MergeRequest::Metrics.arel_table). + on(MergeRequest.arel_table[:id].eq(MergeRequest::Metrics.arel_table[:merge_request_id])) + + if DEPLOYMENT_METRIC_STAGES.include?(name) + # Limit to merge requests that have been deployed to production after `@from` + query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from)) + end + + query + end +end diff --git a/app/models/cycle_analytics/summary.rb b/app/models/cycle_analytics/summary.rb new file mode 100644 index 00000000000..b46db449bf3 --- /dev/null +++ b/app/models/cycle_analytics/summary.rb @@ -0,0 +1,42 @@ +class CycleAnalytics + class Summary + def initialize(project, from:) + @project = project + @from = from + end + + def new_issues + @project.issues.created_after(@from).count + end + + def commits + ref = @project.default_branch.presence + count_commits_for(ref) + end + + def deploys + @project.deployments.where("created_at > ?", @from).count + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits_for(ref) + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + raw_output = IO.popen(cmd) { |io| io.read } + raw_output.lines.count + end + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 1e338889714..91d85c2279b 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -11,7 +11,7 @@ class Deployment < ActiveRecord::Base delegate :name, to: :environment, prefix: true - after_save :keep_around_commit + after_create :create_ref def commit project.commit(sha) @@ -29,17 +29,79 @@ class Deployment < ActiveRecord::Base self == environment.last_deployment end - def keep_around_commit - project.repository.keep_around(self.sha) + def create_ref + project.repository.create_ref(ref, ref_path) end def manual_actions - deployable.try(:other_actions) + @manual_actions ||= deployable.try(:other_actions) end def includes_commit?(commit) return false unless commit - project.repository.is_ancestor?(commit.id, sha) + # Before 8.10, deployments didn't have keep-around refs. Any deployment + # created before then could have a `sha` referring to a commit that no + # longer exists in the repository, so just ignore those. + begin + project.repository.is_ancestor?(commit.id, sha) + rescue Rugged::OdbError + false + end + end + + def update_merge_request_metrics! + return unless environment.update_merge_request_metrics? + + merge_requests = project.merge_requests. + joins(:metrics). + where(target_branch: self.ref, merge_request_metrics: { first_deployed_to_production_at: nil }). + where("merge_request_metrics.merged_at <= ?", self.created_at) + + if previous_deployment + merge_requests = merge_requests.where("merge_request_metrics.merged_at >= ?", previous_deployment.created_at) + end + + # Need to use `map` instead of `select` because MySQL doesn't allow `SELECT`ing from the same table + # that we're updating. + merge_request_ids = + if Gitlab::Database.postgresql? + merge_requests.select(:id) + elsif Gitlab::Database.mysql? + merge_requests.map(&:id) + end + + MergeRequest::Metrics. + where(merge_request_id: merge_request_ids, first_deployed_to_production_at: nil). + update_all(first_deployed_to_production_at: self.created_at) + end + + def previous_deployment + @previous_deployment ||= + project.deployments.joins(:environment). + where(environments: { name: self.environment.name }, ref: self.ref). + where.not(id: self.id). + take + end + + def stop_action + return nil unless on_stop.present? + return nil unless manual_actions + + @stop_action ||= manual_actions.find_by(name: on_stop) + end + + def stoppable? + stop_action.present? + end + + def formatted_deployment_time + created_at.to_time.in_time_zone.to_s(:medium) + end + + private + + def ref_path + File.join(environment.ref_path, 'deployments', iid.to_s) end end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index c816deb4e0c..559b3075905 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -9,17 +9,37 @@ class DiffNote < Note validates :diff_line, presence: true validates :line_code, presence: true, line_code: true validates :noteable_type, inclusion: { in: ['Commit', 'MergeRequest'] } + validates :resolved_by, presence: true, if: :resolved? validate :positions_complete validate :verify_supported + # Keep this scope in sync with the logic in `#resolvable?` + scope :resolvable, -> { user.where(noteable_type: 'MergeRequest') } + scope :resolved, -> { resolvable.where.not(resolved_at: nil) } + scope :unresolved, -> { resolvable.where(resolved_at: nil) } + + after_initialize :ensure_original_discussion_id before_validation :set_original_position, :update_position, on: :create - before_validation :set_line_code + before_validation :set_line_code, :set_original_discussion_id + # We need to do this again, because it's already in `Note`, but is affected by + # `update_position` and needs to run after that. + before_validation :set_discussion_id after_save :keep_around_commits class << self def build_discussion_id(noteable_type, noteable_id, position) [super(noteable_type, noteable_id), *position.key].join("-") end + + # This method must be kept in sync with `#resolve!` + def resolve!(current_user) + unresolved.update_all(resolved_at: Time.now, resolved_by_id: current_user.id) + end + + # This method must be kept in sync with `#unresolve!` + def unresolve! + resolved.update_all(resolved_at: nil, resolved_by_id: nil) + end end def new_diff_note? @@ -30,14 +50,6 @@ class DiffNote < Note { position: position.to_json } end - def discussion_id - @discussion_id ||= self.class.build_discussion_id(noteable_type, noteable_id || commit_id, position) - end - - def original_discussion_id - @original_discussion_id ||= self.class.build_discussion_id(noteable_type, noteable_id || commit_id, original_position) - end - def position=(new_position) if new_position.is_a?(String) new_position = JSON.parse(new_position) rescue nil @@ -63,6 +75,10 @@ class DiffNote < Note diff_file.position(line) == self.original_position end + def original_line_code + self.diff_file.line_code(self.diff_line) + end + def active?(diff_refs = nil) return false unless supported? return true if for_commit? @@ -72,10 +88,47 @@ class DiffNote < Note self.position.diff_refs == diff_refs end + # If you update this method remember to also update the scope `resolvable` + def resolvable? + !system? && for_merge_request? + end + + def resolved? + return false unless resolvable? + + self.resolved_at.present? + end + + # If you update this method remember to also update `.resolve!` + def resolve!(current_user) + return unless resolvable? + return if resolved? + + self.resolved_at = Time.now + self.resolved_by = current_user + save! + end + + # If you update this method remember to also update `.unresolve!` + def unresolve! + return unless resolvable? + return unless resolved? + + self.resolved_at = nil + self.resolved_by = nil + save! + end + + def discussion + return unless resolvable? + + self.noteable.find_diff_discussion(self.discussion_id) + end + private def supported? - !self.for_merge_request? || self.noteable.support_new_diff_notes? + for_commit? || self.noteable.has_complete_diff_refs? end def noteable_diff_refs @@ -94,6 +147,26 @@ class DiffNote < Note self.line_code = self.position.line_code(self.project.repository) end + def ensure_original_discussion_id + return unless self.persisted? + return if self.original_discussion_id + + set_original_discussion_id + update_column(:original_discussion_id, self.original_discussion_id) + end + + def set_original_discussion_id + self.original_discussion_id = Digest::SHA1.hexdigest(build_original_discussion_id) + end + + def build_discussion_id + self.class.build_discussion_id(noteable_type, noteable_id || commit_id, position) + end + + def build_original_discussion_id + self.class.build_discussion_id(noteable_type, noteable_id || commit_id, original_position) + end + def update_position return unless supported? return if for_commit? diff --git a/app/models/discussion.rb b/app/models/discussion.rb index e2218a5f02b..de06c13481a 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -1,7 +1,7 @@ class Discussion NUMBER_OF_TRUNCATED_DIFF_LINES = 16 - attr_reader :first_note, :notes + attr_reader :notes delegate :created_at, :project, @@ -12,12 +12,19 @@ class Discussion :for_merge_request?, :line_code, + :original_line_code, :diff_file, :for_line?, :active?, to: :first_note + delegate :resolved_at, + :resolved_by, + + to: :last_resolved_note, + allow_nil: true + delegate :blob, :highlighted_diff_lines, to: :diff_file, allow_nil: true def self.for_notes(notes) @@ -29,14 +36,29 @@ class Discussion end def initialize(notes) - @first_note = notes.first @notes = notes end + def last_resolved_note + return unless resolved? + + @last_resolved_note ||= resolved_notes.sort_by(&:resolved_at).last + end + + def last_updated_at + last_note.created_at + end + + def last_updated_by + last_note.author + end + def id first_note.discussion_id end + alias_method :to_param, :id + def diff_discussion? first_note.diff_note? end @@ -45,18 +67,78 @@ class Discussion notes.any?(&:legacy_diff_note?) end + def resolvable? + return @resolvable if @resolvable.present? + + @resolvable = diff_discussion? && notes.any?(&:resolvable?) + end + + def resolved? + return @resolved if @resolved.present? + + @resolved = resolvable? && notes.none?(&:to_be_resolved?) + end + + def first_note + @first_note ||= @notes.first + end + + def last_note + @last_note ||= @notes.last + end + + def resolved_notes + notes.select(&:resolved?) + end + + def to_be_resolved? + resolvable? && !resolved? + end + + def can_resolve?(current_user) + return false unless current_user + return false unless resolvable? + + current_user == self.noteable.author || + current_user.can?(:resolve_note, self.project) + end + + def resolve!(current_user) + return unless resolvable? + + update { |notes| notes.resolve!(current_user) } + end + + def unresolve! + return unless resolvable? + + update { |notes| notes.unresolve! } + end + def for_target?(target) self.noteable == target && !diff_discussion? end def active? - return @active if defined?(@active) + return @active if @active.present? @active = first_note.active? end + def collapsed? + return false unless diff_discussion? + + if resolvable? + # New diff discussions only disappear once they are marked resolved + resolved? + else + # Old diff discussions disappear once they become outdated + !active? + end + end + def expanded? - !diff_discussion? || active? + !collapsed? end def reply_attributes @@ -94,4 +176,17 @@ class Discussion prev_lines end + + private + + def update + notes_relation = DiffNote.where(id: notes.map(&:id)).fresh + yield(notes_relation) + + # Set the notes array to the updated notes + @notes = notes_relation.to_a + + # Reset the memoized values + @last_resolved_note = @resolvable = @resolved = @first_note = @last_note = nil + end end diff --git a/app/models/email.rb b/app/models/email.rb index 32a412ab878..826d4f16edb 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -7,10 +7,8 @@ class Email < ActiveRecord::Base validates :email, presence: true, uniqueness: true, email: true validate :unique_email, if: ->(email) { email.email_changed? } - before_validation :cleanup_email - - def cleanup_email - self.email = self.email.downcase.strip + def email=(value) + write_attribute(:email, value.downcase.strip) end def unique_email diff --git a/app/models/environment.rb b/app/models/environment.rb index 75e6f869786..73f415c0ef0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -4,6 +4,7 @@ class Environment < ActiveRecord::Base has_many :deployments before_validation :nullify_external_url + before_save :set_environment_type validates :name, presence: true, @@ -18,6 +19,24 @@ class Environment < ActiveRecord::Base allow_nil: true, addressable_url: true + delegate :stop_action, to: :last_deployment, allow_nil: true + + scope :available, -> { with_state(:available) } + scope :stopped, -> { with_state(:stopped) } + + state_machine :state, initial: :available do + event :start do + transition stopped: :available + end + + event :stop do + transition available: :stopped + end + + state :available + state :stopped + end + def last_deployment deployments.last end @@ -26,9 +45,53 @@ class Environment < ActiveRecord::Base self.external_url = nil if self.external_url.blank? end + def set_environment_type + names = name.split('/') + + self.environment_type = + if names.many? + names.first + else + nil + end + end + def includes_commit?(commit) return false unless last_deployment last_deployment.includes_commit?(commit) end + + def update_merge_request_metrics? + self.name == "production" + end + + def first_deployment_for(commit) + ref = project.repository.ref_name_for_sha(ref_path, commit.sha) + + return nil unless ref + + deployment_iid = ref.split('/').last + deployments.find_by(iid: deployment_iid) + end + + def ref_path + "refs/environments/#{Shellwords.shellescape(name)}" + end + + def formatted_external_url + return nil unless external_url + + external_url.gsub(/\A.*?:\/\//, '') + end + + def stoppable? + available? && stop_action.present? + end + + def stop!(current_user) + return unless stoppable? + + stop_action.play(current_user) + end end diff --git a/app/models/event.rb b/app/models/event.rb index fd736d12359..c76d88b1c7b 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,6 +1,6 @@ class Event < ActiveRecord::Base include Sortable - default_scope { where.not(author_id: nil) } + default_scope { reorder(nil).where.not(author_id: nil) } CREATED = 1 UPDATED = 2 @@ -12,6 +12,9 @@ class Event < ActiveRecord::Base JOINED = 8 # User joined project LEFT = 9 # User left project DESTROYED = 10 + EXPIRED = 11 # User left project due to expiry + + RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour delegate :name, :email, to: :author, prefix: true, allow_nil: true delegate :title, to: :issue, prefix: true, allow_nil: true @@ -46,6 +49,7 @@ class Event < ActiveRecord::Base update_all(updated_at: Time.now) end + # Update Gitlab::ContributionsCalendar#activity_dates if this changes def contributions where("action = ? OR (target_type in (?) AND action in (?))", Event::PUSHED, ["MergeRequest", "Issue"], @@ -59,15 +63,17 @@ class Event < ActiveRecord::Base def visible_to_user?(user = nil) if push? - true + Ability.allowed?(user, :download_code, project) elsif membership_changed? true elsif created_project? true elsif issue? || issue_note? - Ability.abilities.allowed?(user, :read_issue, note? ? note_target : target) + Ability.allowed?(user, :read_issue, note? ? note_target : target) + elsif merge_request? || merge_request_note? + Ability.allowed?(user, :read_merge_request, note? ? note_target : target) else - ((merge_request? || note?) && target.present?) || milestone? + milestone? end end @@ -111,6 +117,10 @@ class Event < ActiveRecord::Base action == LEFT end + def expired? + action == EXPIRED + end + def destroyed? action == DESTROYED end @@ -120,7 +130,7 @@ class Event < ActiveRecord::Base end def membership_changed? - joined? || left? + joined? || left? || expired? end def created_project? @@ -180,6 +190,8 @@ class Event < ActiveRecord::Base 'joined' elsif left? 'left' + elsif expired? + 'removed due to membership expiration from' elsif destroyed? 'destroyed' elsif commented? @@ -278,6 +290,10 @@ class Event < ActiveRecord::Base note? && target && target.for_issue? end + def merge_request_note? + note? && target && target.for_merge_request? + end + def project_snippet_note? target.for_snippet? end @@ -324,8 +340,22 @@ class Event < ActiveRecord::Base end def reset_project_activity - if project && Gitlab::ExclusiveLease.new("project:update_last_activity_at:#{project.id}", timeout: 60).try_obtain - project.update_column(:last_activity_at, self.created_at) - end + return unless project + + # Don't bother updating if we know the project was updated recently. + return if recent_update? + + # At this point it's possible for multiple threads/processes to try to + # update the project. Only one query should actually perform the update, + # hence we add the extra WHERE clause for last_activity_at. + Project.unscoped.where(id: project_id). + where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago). + update_all(last_activity_at: created_at) + end + + private + + def recent_update? + project.last_activity_at > RESET_PROJECT_ACTIVITY_INTERVAL.ago end end diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index b7894c99846..91b508eb325 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -29,6 +29,10 @@ class ExternalIssue @project end + def project_id + @project.id + end + # Pattern used to extract `JIRA-123` issue references from text def self.reference_pattern @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} diff --git a/app/models/global_label.rb b/app/models/global_label.rb index ddd4bad5c21..698a7bbd327 100644 --- a/app/models/global_label.rb +++ b/app/models/global_label.rb @@ -4,6 +4,10 @@ class GlobalLabel delegate :color, :description, to: :@first_label + def for_display + @first_label + end + def self.build_collection(labels) labels = labels.group_by(&:title) diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index da7c265a371..cde4a568577 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -4,11 +4,16 @@ class GlobalMilestone attr_accessor :title, :milestones alias_attribute :name, :title + def for_display + @first_milestone + end + def self.build_collection(milestones) milestones = milestones.group_by(&:title) milestones.map do |title, milestones| - new(title, milestones) + milestones_relation = Milestone.where(id: milestones.map(&:id)) + new(title, milestones_relation) end end @@ -16,6 +21,7 @@ class GlobalMilestone @title = title @name = title @milestones = milestones + @first_milestone = milestones.find {|m| m.description.present? } || milestones.first end def safe_title @@ -31,7 +37,7 @@ class GlobalMilestone end def projects - @projects ||= Project.for_milestones(milestones.map(&:id)) + @projects ||= Project.for_milestones(milestones.select(:id)) end def state @@ -53,19 +59,19 @@ class GlobalMilestone end def issues - @issues ||= Issue.of_milestones(milestones.map(&:id)).includes(:project) + @issues ||= Issue.of_milestones(milestones.select(:id)).includes(:project, :assignee, :labels) end def merge_requests - @merge_requests ||= MergeRequest.of_milestones(milestones.map(&:id)).includes(:target_project) + @merge_requests ||= MergeRequest.of_milestones(milestones.select(:id)).includes(:target_project, :assignee, :labels) end def participants - @participants ||= milestones.map(&:participants).flatten.compact.uniq + @participants ||= milestones.includes(:participants).map(&:participants).flatten.compact.uniq end def labels - @labels ||= GlobalLabel.build_collection(milestones.map(&:labels).flatten) + @labels ||= GlobalLabel.build_collection(milestones.includes(:labels).map(&:labels).flatten) .sort_by!(&:title) end diff --git a/app/models/group.rb b/app/models/group.rb index 37631b99701..d9e90cd256a 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -6,7 +6,7 @@ class Group < Namespace include AccessRequestable include Referable - has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' + has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :group_members has_many :users, through: :group_members has_many :owners, @@ -19,6 +19,7 @@ class Group < Namespace has_many :project_group_links, dependent: :destroy has_many :shared_projects, through: :project_group_links, source: :project has_many :notification_settings, dependent: :destroy, as: :source + has_many :labels, class_name: 'GroupLabel' validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :visibility_level_allowed_by_projects @@ -67,7 +68,7 @@ class Group < Namespace end def web_url - Gitlab::Routing.url_helpers.group_url(self) + Gitlab::Routing.url_helpers.group_canonical_url(self) end def human_name @@ -95,34 +96,51 @@ class Group < Namespace end end - def add_users(user_ids, access_level, current_user = nil) - user_ids.each do |user_id| - Member.add_user(self.group_members, user_id, access_level, current_user) - end + def lfs_enabled? + return false unless Gitlab.config.lfs.enabled + return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? + + self[:lfs_enabled] + end + + def add_users(users, access_level, current_user: nil, expires_at: nil) + GroupMember.add_users_to_group( + self, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) end - def add_user(user, access_level, current_user = nil) - add_users([user], access_level, current_user) + def add_user(user, access_level, current_user: nil, expires_at: nil) + GroupMember.add_user( + self, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) end def add_guest(user, current_user = nil) - add_user(user, Gitlab::Access::GUEST, current_user) + add_user(user, :guest, current_user: current_user) end def add_reporter(user, current_user = nil) - add_user(user, Gitlab::Access::REPORTER, current_user) + add_user(user, :reporter, current_user: current_user) end def add_developer(user, current_user = nil) - add_user(user, Gitlab::Access::DEVELOPER, current_user) + add_user(user, :developer, current_user: current_user) end def add_master(user, current_user = nil) - add_user(user, Gitlab::Access::MASTER, current_user) + add_user(user, :master, current_user: current_user) end def add_owner(user, current_user = nil) - add_user(user, Gitlab::Access::OWNER, current_user) + add_user(user, :owner, current_user: current_user) end def has_owner?(user) diff --git a/app/models/group_label.rb b/app/models/group_label.rb new file mode 100644 index 00000000000..68841ace2e6 --- /dev/null +++ b/app/models/group_label.rb @@ -0,0 +1,15 @@ +class GroupLabel < Label + belongs_to :group + + validates :group, presence: true + + alias_attribute :subject, :group + + def subject_foreign_key + 'group_id' + end + + def to_reference(source_project = nil, target_project = nil, format: :id) + super(source_project, target_project, format: format) + end +end diff --git a/app/models/guest.rb b/app/models/guest.rb new file mode 100644 index 00000000000..01285ca1264 --- /dev/null +++ b/app/models/guest.rb @@ -0,0 +1,7 @@ +class Guest + class << self + def can?(action, subject) + Ability.allowed?(nil, action, subject) + end + end +end diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 836a75b0608..c631e7a7df5 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -2,6 +2,7 @@ class ProjectHook < WebHook belongs_to :project scope :issue_hooks, -> { where(issues_events: true) } + scope :confidential_issue_hooks, -> { where(confidential_issues_events: true) } scope :note_hooks, -> { where(note_events: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true) } scope :build_hooks, -> { where(build_events: true) } diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index f365dee3141..595602e80fe 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -4,6 +4,7 @@ class WebHook < ActiveRecord::Base default_value_for :push_events, true default_value_for :issues_events, false + default_value_for :confidential_issues_events, false default_value_for :note_events, false default_value_for :merge_requests_events, false default_value_for :tag_push_events, false diff --git a/app/models/issue.rb b/app/models/issue.rb index 788611305fe..adbca510ef7 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -5,7 +5,6 @@ class Issue < ActiveRecord::Base include Issuable include Referable include Sortable - include Taskable include Spammable include FasterCacheKeys @@ -23,6 +22,8 @@ class Issue < ActiveRecord::Base has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + validates :project, presence: true scope :cared, ->(user) { where(assignee_id: user) } @@ -36,6 +37,8 @@ class Issue < ActiveRecord::Base scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } + scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } + attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -134,6 +137,10 @@ class Issue < ActiveRecord::Base reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end + def self.project_foreign_key + 'project_id' + end + def self.sort(method, excluded_labels: []) case method.to_s when 'due_date_asc' then order_due_date_asc @@ -203,7 +210,13 @@ class Issue < ActiveRecord::Base note.all_references(current_user, extractor: ext) end - ext.merge_requests.select { |mr| mr.open? && mr.closes_issue?(self) } + merge_requests = ext.merge_requests.select(&:open?) + if merge_requests.any? + ids = MergeRequestsClosingIssues.where(merge_request_id: merge_requests.map(&:id), issue_id: id).pluck(:merge_request_id) + merge_requests.select { |mr| mr.id.in?(ids) } + else + [] + end end def moved? @@ -237,10 +250,41 @@ class Issue < ActiveRecord::Base # Returns `true` if the current issue can be viewed by either a logged in User # or an anonymous user. def visible_to_user?(user = nil) + return false unless project.feature_available?(:issues, user) + user ? readable_by?(user) : publicly_visible? end + def overdue? + due_date.try(:past?) || false + end + + # Only issues on public projects should be checked for spam + def check_for_spam? + project.public? + end + + def as_json(options = {}) + super(options).tap do |json| + json[:subscribed] = subscribed?(options[:user]) if options.has_key?(:user) + + if options.has_key?(:labels) + json[:labels] = labels.as_json( + project: project, + only: [:id, :title, :description, :color, :priority], + methods: [:text_color] + ) + end + end + end + + private + # Returns `true` if the given User can read the current Issue. + # + # This method duplicates the same check of issue_policy.rb + # for performance reasons, check commit: 002ad215818450d2cbbc5fa065850a953dc7ada8 + # Make sure to sync this method with issue_policy.rb def readable_by?(user) if user.admin? true @@ -261,13 +305,4 @@ class Issue < ActiveRecord::Base def publicly_visible? project.public? && !confidential? end - - def overdue? - due_date.try(:past?) || false - end - - # Only issues on public projects should be checked for spam - def check_for_spam? - project.public? - end end diff --git a/app/models/issue/metrics.rb b/app/models/issue/metrics.rb new file mode 100644 index 00000000000..012d545c440 --- /dev/null +++ b/app/models/issue/metrics.rb @@ -0,0 +1,21 @@ +class Issue::Metrics < ActiveRecord::Base + belongs_to :issue + + def record! + if issue.milestone_id.present? && self.first_associated_with_milestone_at.blank? + self.first_associated_with_milestone_at = Time.now + end + + if issue_assigned_to_list_label? && self.first_added_to_board_at.blank? + self.first_added_to_board_at = Time.now + end + + self.save + end + + private + + def issue_assigned_to_list_label? + issue.labels.any? { |label| label.lists.present? } + end +end diff --git a/app/models/issue_collection.rb b/app/models/issue_collection.rb new file mode 100644 index 00000000000..f0b7d9914c8 --- /dev/null +++ b/app/models/issue_collection.rb @@ -0,0 +1,42 @@ +# IssueCollection can be used to reduce a list of issues down to a subset. +# +# IssueCollection is not meant to be some sort of Enumerable, instead it's meant +# to take a list of issues and return a new list of issues based on some +# criteria. For example, given a list of issues you may want to return a list of +# issues that can be read or updated by a given user. +class IssueCollection + attr_reader :collection + + def initialize(collection) + @collection = collection + end + + # Returns all the issues that can be updated by the user. + def updatable_by_user(user) + return collection if user.admin? + + # Given all the issue projects we get a list of projects that the current + # user has at least reporter access to. + projects_with_reporter_access = user. + projects_with_reporter_access_limited_to(project_ids). + pluck(:id) + + collection.select do |issue| + if projects_with_reporter_access.include?(issue.project_id) + true + elsif issue.is_a?(Issue) + issue.assignee_or_author?(user) + else + false + end + end + end + + alias_method :visible_to, :updatable_by_user + + private + + def project_ids + @project_ids ||= collection.map(&:project_id).uniq + end +end diff --git a/app/models/label.rb b/app/models/label.rb index 35e678001dc..d9287f2dc29 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -1,4 +1,5 @@ class Label < ActiveRecord::Base + include CacheMarkdownField include Referable include Subscribable @@ -8,36 +9,55 @@ class Label < ActiveRecord::Base None = LabelStruct.new('No Label', 'No Label') Any = LabelStruct.new('Any Label', '') + cache_markdown_field :description, pipeline: :single_line + DEFAULT_COLOR = '#428BCA' default_value_for :color, DEFAULT_COLOR - belongs_to :project + has_many :lists, dependent: :destroy + has_many :priorities, class_name: 'LabelPriority' has_many :label_links, dependent: :destroy has_many :issues, through: :label_links, source: :target, source_type: 'Issue' has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' validates :color, color: true, allow_blank: false - validates :project, presence: true, unless: Proc.new { |service| service.template? } # Don't allow ',' for label titles - validates :title, - presence: true, - format: { with: /\A[^,]+\z/ }, - uniqueness: { scope: :project_id } - - before_save :nullify_priority + validates :title, presence: true, format: { with: /\A[^,]+\z/ } + validates :title, uniqueness: { scope: [:group_id, :project_id] } default_scope { order(title: :asc) } - scope :templates, -> { where(template: true) } + scope :templates, -> { where(template: true) } + scope :with_title, ->(title) { where(title: title) } - def self.prioritized - where.not(priority: nil).reorder(:priority, :title) + def self.prioritized(project) + joins(:priorities) + .where(label_priorities: { project_id: project }) + .reorder('label_priorities.priority ASC, labels.title ASC') end - def self.unprioritized - where(priority: nil) + def self.unprioritized(project) + labels = Label.arel_table + priorities = LabelPriority.arel_table + + label_priorities = labels.join(priorities, Arel::Nodes::OuterJoin). + on(labels[:id].eq(priorities[:label_id]).and(priorities[:project_id].eq(project.id))). + join_sources + + joins(label_priorities).where(priorities[:priority].eq(nil)) + end + + def self.left_join_priorities + labels = Label.arel_table + priorities = LabelPriority.arel_table + + label_priorities = labels.join(priorities, Arel::Nodes::OuterJoin). + on(labels[:id].eq(priorities[:label_id])). + join_sources + + joins(label_priorities) end alias_attribute :name, :title @@ -72,6 +92,51 @@ class Label < ActiveRecord::Base nil end + def open_issues_count(user = nil) + issues_count(user, state: 'opened') + end + + def closed_issues_count(user = nil) + issues_count(user, state: 'closed') + end + + def open_merge_requests_count(user = nil) + params = { + subject_foreign_key => subject.id, + label_name: title, + scope: 'all', + state: 'opened' + } + + MergeRequestsFinder.new(user, params.with_indifferent_access).execute.count + end + + def prioritize!(project, value) + label_priority = priorities.find_or_initialize_by(project_id: project.id) + label_priority.priority = value + label_priority.save! + end + + def unprioritize!(project) + priorities.where(project: project).delete_all + end + + def priority(project) + priorities.find_by(project: project).try(:priority) + end + + def template? + template + end + + def text_color + LabelsHelper.text_color_for_bg(self.color) + end + + def title=(value) + write_attribute(:title, sanitize_title(value)) if value.present? + end + ## # Returns the String necessary to reference this Label in Markdown # @@ -79,49 +144,40 @@ class Label < ActiveRecord::Base # # Examples: # - # Label.first.to_reference # => "~1" - # Label.first.to_reference(format: :name) # => "~\"bug\"" - # Label.first.to_reference(project) # => "gitlab-org/gitlab-ce~1" + # Label.first.to_reference # => "~1" + # Label.first.to_reference(format: :name) # => "~\"bug\"" + # Label.first.to_reference(project1, project2) # => "gitlab-org/gitlab-ce~1" # # Returns a String # - def to_reference(from_project = nil, format: :id) + def to_reference(source_project = nil, target_project = nil, format: :id) format_reference = label_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if cross_project_reference?(from_project) - project.to_reference + reference + if cross_project_reference?(source_project, target_project) + source_project.to_reference + reference else reference end end - def open_issues_count(user = nil) - issues.visible_to_user(user).opened.count - end - - def closed_issues_count(user = nil) - issues.visible_to_user(user).closed.count - end - - def open_merge_requests_count - merge_requests.opened.count + def as_json(options = {}) + super(options).tap do |json| + json[:priority] = priority(options[:project]) if options.has_key?(:project) + end end - def template? - template - end + private - def text_color - LabelsHelper::text_color_for_bg(self.color) + def cross_project_reference?(source_project, target_project) + source_project && target_project && source_project != target_project end - def title=(value) - write_attribute(:title, sanitize_title(value)) if value.present? + def issues_count(user, params = {}) + params.merge!(subject_foreign_key => subject.id, label_name: title, scope: 'all') + IssuesFinder.new(user, params.with_indifferent_access).execute.count end - private - def label_format_reference(format = :id) raise StandardError, 'Unknown format' unless [:id, :name].include?(format) @@ -132,10 +188,6 @@ class Label < ActiveRecord::Base end end - def nullify_priority - self.priority = nil if priority.blank? - end - def sanitize_title(value) CGI.unescapeHTML(Sanitize.clean(value.to_s)) end diff --git a/app/models/label_priority.rb b/app/models/label_priority.rb new file mode 100644 index 00000000000..5b85e0b6533 --- /dev/null +++ b/app/models/label_priority.rb @@ -0,0 +1,8 @@ +class LabelPriority < ActiveRecord::Base + belongs_to :project + belongs_to :label + + validates :project, :label, :priority, presence: true + validates :label_id, uniqueness: { scope: :project_id } + validates :priority, numericality: { only_integer: true, greater_than_or_equal_to: 0 } +end diff --git a/app/models/legacy_diff_note.rb b/app/models/legacy_diff_note.rb index 6ed66001513..40277a9b139 100644 --- a/app/models/legacy_diff_note.rb +++ b/app/models/legacy_diff_note.rb @@ -8,8 +8,8 @@ class LegacyDiffNote < Note before_create :set_diff class << self - def build_discussion_id(noteable_type, noteable_id, line_code, active = true) - [super(noteable_type, noteable_id), line_code, active].join("-") + def build_discussion_id(noteable_type, noteable_id, line_code) + [super(noteable_type, noteable_id), line_code].join("-") end end @@ -21,10 +21,6 @@ class LegacyDiffNote < Note { line_code: line_code } end - def discussion_id - @discussion_id ||= self.class.build_discussion_id(noteable_type, noteable_id || commit_id, line_code) - end - def project_repository if RequestStore.active? RequestStore.fetch("project:#{project_id}:repository") { self.project.repository } @@ -53,6 +49,10 @@ class LegacyDiffNote < Note !line.meta? && diff_file.line_code(line) == self.line_code end + def original_line_code + self.line_code + end + # Check if this note is part of an "active" discussion # # This will always return true for anything except MergeRequest noteables, @@ -119,4 +119,8 @@ class LegacyDiffNote < Note diffs = noteable.raw_diffs(Commit.max_diff_options) diffs.find { |d| d.new_path == self.diff.new_path } end + + def build_discussion_id + self.class.build_discussion_id(noteable_type, noteable_id || commit_id, line_code) + end end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 18657c3e1c8..7712d5783e0 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -17,4 +17,10 @@ class LfsObject < ActiveRecord::Base def project_allowed_access?(project) projects.exists?(storage_project(project).id) end + + def self.destroy_unreferenced + joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") + .where(lfs_objects_projects: { id: nil }) + .destroy_all + end end diff --git a/app/models/list.rb b/app/models/list.rb new file mode 100644 index 00000000000..065d75bd1dc --- /dev/null +++ b/app/models/list.rb @@ -0,0 +1,45 @@ +class List < ActiveRecord::Base + belongs_to :board + belongs_to :label + + enum list_type: { backlog: 0, label: 1, done: 2 } + + validates :board, :list_type, presence: true + validates :label, :position, presence: true, if: :label? + validates :label_id, uniqueness: { scope: :board_id }, if: :label? + validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :label? + + before_destroy :can_be_destroyed + + scope :destroyable, -> { where(list_type: list_types[:label]) } + scope :movable, -> { where(list_type: list_types[:label]) } + + def destroyable? + label? + end + + def movable? + label? + end + + def title + label? ? label.name : list_type.humanize + end + + def as_json(options = {}) + super(options).tap do |json| + if options.has_key?(:label) + json[:label] = label.as_json( + project: board.project, + only: [:id, :title, :description, :color] + ) + end + end + end + + private + + def can_be_destroyed + destroyable? + end +end diff --git a/app/models/member.rb b/app/models/member.rb index 24ab1276ee9..b89ba8ecbb8 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,6 +1,7 @@ class Member < ActiveRecord::Base include Sortable include Importable + include Expirable include Gitlab::Access attr_accessor :raw_invite_token @@ -27,17 +28,34 @@ class Member < ActiveRecord::Base allow_nil: true } + # This scope encapsulates (most of) the conditions a row in the member table + # must satisfy if it is a valid permission. Of particular note: + # + # * Access requests must be excluded + # * Blocked users must be excluded + # * Invitations take effect immediately + # * expires_at is not implemented. A background worker purges expired rows + scope :active, -> do + is_external_invite = arel_table[:user_id].eq(nil).and(arel_table[:invite_token].not_eq(nil)) + user_is_active = User.arel_table[:state].eq(:active) + + includes(:user).references(:users) + .where(is_external_invite.or(user_is_active)) + .where(requested_at: nil) + end + scope :invite, -> { where.not(invite_token: nil) } scope :non_invite, -> { where(invite_token: nil) } scope :request, -> { where.not(requested_at: nil) } - scope :has_access, -> { where('access_level > 0') } - scope :guests, -> { where(access_level: GUEST) } - scope :reporters, -> { where(access_level: REPORTER) } - scope :developers, -> { where(access_level: DEVELOPER) } - scope :masters, -> { where(access_level: MASTER) } - scope :owners, -> { where(access_level: OWNER) } - scope :owners_and_masters, -> { where(access_level: [OWNER, MASTER]) } + scope :has_access, -> { active.where('access_level > 0') } + + scope :guests, -> { active.where(access_level: GUEST) } + scope :reporters, -> { active.where(access_level: REPORTER) } + scope :developers, -> { active.where(access_level: DEVELOPER) } + scope :masters, -> { active.where(access_level: MASTER) } + scope :owners, -> { active.where(access_level: OWNER) } + scope :owners_and_masters, -> { active.where(access_level: [OWNER, MASTER]) } before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? } @@ -62,48 +80,75 @@ class Member < ActiveRecord::Base find_by(invite_token: invite_token) end - # This method is used to find users that have been entered into the "Add members" field. - # These can be the User objects directly, their IDs, their emails, or new emails to be invited. - def user_for_id(user_id) - return user_id if user_id.is_a?(User) - - user = User.find_by(id: user_id) - user ||= User.find_by(email: user_id) - user ||= user_id - user - end - - def add_user(members, user_id, access_level, current_user = nil) - user = user_for_id(user_id) + def add_user(source, user, access_level, current_user: nil, expires_at: nil) + user = retrieve_user(user) + access_level = retrieve_access_level(access_level) # `user` can be either a User object or an email to be invited - if user.is_a?(User) - member = members.find_or_initialize_by(user_id: user.id) + member = + if user.is_a?(User) + source.members.find_by(user_id: user.id) || + source.requesters.find_by(user_id: user.id) || + source.members.build(user_id: user.id) + else + source.members.build(invite_email: user) + end + + return member unless can_update_member?(current_user, member) + + member.attributes = { + created_by: member.created_by || current_user, + access_level: access_level, + expires_at: expires_at + } + + if member.request? + ::Members::ApproveAccessRequestService.new( + source, + current_user, + id: member.id, + access_level: access_level + ).execute else - member = members.build - member.invite_email = user + member.save end - if can_update_member?(current_user, member) || project_creator?(member, access_level) - member.created_by ||= current_user - member.access_level = access_level + member + end - member.save - end + def access_levels + Gitlab::Access.sym_options end private + # This method is used to find users that have been entered into the "Add members" field. + # These can be the User objects directly, their IDs, their emails, or new emails to be invited. + def retrieve_user(user) + return user if user.is_a?(User) + + User.find_by(id: user) || User.find_by(email: user) || user + end + + def retrieve_access_level(access_level) + access_levels.fetch(access_level) { access_level.to_i } + end + def can_update_member?(current_user, member) # There is no current user for bulk actions, in which case anything is allowed - !current_user || - current_user.can?(:update_group_member, member) || - current_user.can?(:update_project_member, member) + !current_user || current_user.can?(:"update_#{member.type.underscore}", member) end - def project_creator?(member, access_level) - member.new_record? && member.owner? && - access_level.to_i == ProjectMember::MASTER + def add_users_to_source(source, users, access_level, current_user: nil, expires_at: nil) + users.each do |user| + add_user( + source, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) + end end end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 2f13d339c89..204f34f0269 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -1,7 +1,7 @@ class GroupMember < Member SOURCE_TYPE = 'Namespace' - belongs_to :group, class_name: 'Group', foreign_key: 'source_id' + belongs_to :group, foreign_key: 'source_id' # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE @@ -12,6 +12,22 @@ class GroupMember < Member Gitlab::Access.options_with_owner end + def self.access_levels + Gitlab::Access.sym_options_with_owner + end + + def self.add_users_to_group(group, users, access_level, current_user: nil, expires_at: nil) + self.transaction do + add_users_to_source( + group, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) + end + end + def group source end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 18e97c969d7..008fff0857c 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -3,7 +3,7 @@ class ProjectMember < Member include Gitlab::ShellAdapter - belongs_to :project, class_name: 'Project', foreign_key: 'source_id' + belongs_to :project, foreign_key: 'source_id' # Make sure project member points only to project as it source default_value_for :source_type, SOURCE_TYPE @@ -34,30 +34,20 @@ class ProjectMember < Member # :master # ) # - def add_users_to_projects(project_ids, user_ids, access, current_user = nil) - access_level = if roles_hash.has_key?(access) - roles_hash[access] - elsif roles_hash.values.include?(access.to_i) - access - else - raise "Non valid access" - end - - users = user_ids.map { |user_id| Member.user_for_id(user_id) } - - ProjectMember.transaction do + def add_users_to_projects(project_ids, users, access_level, current_user: nil, expires_at: nil) + self.transaction do project_ids.each do |project_id| project = Project.find(project_id) - users.each do |user| - Member.add_user(project.project_members, user, access_level, current_user) - end + add_users_to_source( + project, + users, + access_level, + current_user: current_user, + expires_at: expires_at + ) end end - - true - rescue - false end def truncate_teams(project_ids) @@ -78,13 +68,15 @@ class ProjectMember < Member truncate_teams [project.id] end - def roles_hash - Gitlab::Access.sym_options - end - def access_level_roles Gitlab::Access.options end + + private + + def can_update_member?(current_user, member) + super || (member.owner? && member.new_record?) + end end def access_field @@ -129,7 +121,11 @@ class ProjectMember < Member end def post_destroy_hook - event_service.leave_project(self.project, self.user) + if expired? + event_service.expired_leave_project(self.project, self.user) + else + event_service.leave_project(self.project, self.user) + end super end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index fe799382fd0..d76feb9680e 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -3,21 +3,24 @@ class MergeRequest < ActiveRecord::Base include Issuable include Referable include Sortable - include Taskable include Importable - belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project" - belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project" + belongs_to :target_project, class_name: "Project" + belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" - has_one :merge_request_diff, dependent: :destroy + has_many :merge_request_diffs, dependent: :destroy + has_one :merge_request_diff, + -> { order('merge_request_diffs.id DESC') } has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + serialize :merge_params, Hash - after_create :create_merge_request_diff, unless: :importing? - after_update :update_merge_request_diff + after_create :ensure_merge_request_diff, unless: :importing? + after_update :reload_diff_if_branch_changed delegate :commits, :real_size, to: :merge_request_diff, prefix: nil @@ -27,7 +30,7 @@ class MergeRequest < ActiveRecord::Base # Temporary fields to store compare vars # when creating new merge request - attr_accessor :can_be_created, :compare_commits, :compare + attr_accessor :can_be_created, :compare_commits, :diff_options, :compare state_machine :state, initial: :opened do event :close do @@ -89,13 +92,13 @@ class MergeRequest < ActiveRecord::Base end end - validates :source_project, presence: true, unless: [:allow_broken, :importing?] + validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_without_fork?] validates :source_branch, presence: true validates :target_project, presence: true validates :target_branch, presence: true validates :merge_user, presence: true, if: :merge_when_build_succeeds? - validate :validate_branches, unless: [:allow_broken, :importing?] - validate :validate_fork + validate :validate_branches, unless: [:allow_broken, :importing?, :closed_without_fork?] + validate :validate_fork, unless: :closed_without_fork? scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) } scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) } @@ -133,6 +136,10 @@ class MergeRequest < ActiveRecord::Base reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end + def self.project_foreign_key + 'target_project_id' + end + # Returns all the merge requests from an ActiveRecord:Relation. # # This method uses a UNION as it usually operates on the result of @@ -151,6 +158,20 @@ class MergeRequest < ActiveRecord::Base where("merge_requests.id IN (#{union.to_sql})") end + WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze + + def self.work_in_progress?(title) + !!(title =~ WIP_REGEX) + end + + def self.wipless_title(title) + title.sub(WIP_REGEX, "") + end + + def self.wip_title(title) + work_in_progress?(title) ? title : "WIP: #{title}" + end + def to_reference(from_project = nil) reference = "#{self.class.reference_prefix}#{iid}" @@ -170,22 +191,22 @@ class MergeRequest < ActiveRecord::Base end def diffs(diff_options = nil) - if self.compare - self.compare.diffs(diff_options) + if compare + compare.diffs(diff_options) else - Gitlab::Diff::FileCollection::MergeRequest.new(self, diff_options: diff_options) + merge_request_diff.diffs(diff_options) end end def diff_size - merge_request_diff.size + diffs(diff_options).size end def diff_base_commit if persisted? merge_request_diff.base_commit - elsif diff_start_commit && diff_head_commit - self.target_project.merge_base_commit(diff_start_sha, diff_head_sha) + else + branch_merge_base_commit end end @@ -238,12 +259,21 @@ class MergeRequest < ActiveRecord::Base def source_branch_head source_branch_ref = @source_branch_sha || source_branch - source_project.repository.commit(source_branch) if source_branch_ref + source_project.repository.commit(source_branch_ref) if source_branch_ref end def target_branch_head target_branch_ref = @target_branch_sha || target_branch - target_project.repository.commit(target_branch) if target_branch_ref + target_project.repository.commit(target_branch_ref) if target_branch_ref + end + + def branch_merge_base_commit + start_sha = target_branch_sha + head_sha = source_branch_sha + + if start_sha && head_sha + target_project.merge_base_commit(start_sha, head_sha) + end end def target_branch_sha @@ -267,16 +297,16 @@ class MergeRequest < ActiveRecord::Base # Return diff_refs instance trying to not touch the git repository def diff_sha_refs if merge_request_diff && merge_request_diff.diff_refs_by_sha? - return Gitlab::Diff::DiffRefs.new( - base_sha: merge_request_diff.base_commit_sha, - start_sha: merge_request_diff.start_commit_sha, - head_sha: merge_request_diff.head_commit_sha - ) + merge_request_diff.diff_refs else diff_refs end end + def branch_merge_base_sha + branch_merge_base_commit.try(:sha) + end + def validate_branches if target_project == source_project && target_branch == source_branch errors.add :branch_conflict, "You can not use same project/branch for source and target" @@ -294,36 +324,53 @@ class MergeRequest < ActiveRecord::Base def validate_fork return true unless target_project && source_project + return true if target_project == source_project + return true unless source_project_missing? - if target_project == source_project - true - else - # If source and target projects are different - # we should check if source project is actually a fork of target project - if source_project.forked_from?(target_project) - true - else - errors.add :validate_fork, - 'Source project is not a fork of target project' - end - end + errors.add :validate_fork, + 'Source project is not a fork of the target project' end - def update_merge_request_diff + def closed_without_fork? + closed? && source_project_missing? + end + + def source_project_missing? + return false unless for_fork? + return true unless source_project + + !source_project.forked_from?(target_project) + end + + def reopenable? + closed? && !source_project_missing? && source_branch_exists? + end + + def ensure_merge_request_diff + merge_request_diff || create_merge_request_diff + end + + def create_merge_request_diff + merge_request_diffs.create + reload_merge_request_diff + end + + def reload_merge_request_diff + merge_request_diff(true) + end + + def reload_diff_if_branch_changed if source_branch_changed? || target_branch_changed? reload_diff end end def reload_diff - return unless merge_request_diff && open? + return unless open? old_diff_refs = self.diff_refs - - merge_request_diff.reload_content - + create_merge_request_diff MergeRequests::MergeRequestDiffCacheService.new.execute(self) - new_diff_refs = self.diff_refs update_diff_notes_positions( @@ -353,14 +400,16 @@ class MergeRequest < ActiveRecord::Base @closed_event ||= target_project.events.where(target_id: self.id, target_type: "MergeRequest", action: Event::CLOSED).last end - WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze - def work_in_progress? - !!(title =~ WIP_REGEX) + self.class.work_in_progress?(title) end def wipless_title - self.title.sub(WIP_REGEX, "") + self.class.wipless_title(self.title) + end + + def wip_title + self.class.wip_title(self.title) end def mergeable?(skip_ci_check: false) @@ -376,6 +425,7 @@ class MergeRequest < ActiveRecord::Base return false if work_in_progress? return false if broken? return false unless skip_ci_check || mergeable_ci_state? + return false unless mergeable_discussions_state? true end @@ -387,16 +437,16 @@ class MergeRequest < ActiveRecord::Base def can_remove_source_branch?(current_user) !source_project.protected_branch?(source_branch) && !source_project.root_ref?(source_branch) && - Ability.abilities.allowed?(current_user, :push_code, source_project) && + Ability.allowed?(current_user, :push_code, source_project) && diff_head_commit == source_branch_head end def should_remove_source_branch? - merge_params['should_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['should_remove_source_branch']) end def force_remove_source_branch? - merge_params['force_remove_source_branch'].present? + Gitlab::Utils.to_boolean(merge_params['force_remove_source_branch']) end def remove_source_branch? @@ -418,6 +468,38 @@ class MergeRequest < ActiveRecord::Base ) end + def discussions + @discussions ||= self.mr_and_commit_notes. + inc_relations_for_view. + fresh. + discussions + end + + def diff_discussions + @diff_discussions ||= self.notes.diff_notes.discussions + end + + def find_diff_discussion(discussion_id) + notes = self.notes.diff_notes.where(discussion_id: discussion_id).fresh.to_a + return if notes.empty? + + Discussion.new(notes) + end + + def discussions_resolvable? + diff_discussions.any?(&:resolvable?) + end + + def discussions_resolved? + discussions_resolvable? && diff_discussions.none?(&:to_be_resolved?) + end + + def mergeable_discussions_state? + return true unless project.only_allow_merge_if_all_discussions_are_resolved? + + discussions_resolved? + end + def hook_attrs attrs = { source: source_project.try(:hook_attrs), @@ -441,6 +523,23 @@ class MergeRequest < ActiveRecord::Base target_project end + # If the merge request closes any issues, save this information in the + # `MergeRequestsClosingIssues` model. This is a performance optimization. + # Calculating this information for a number of merge requests requires + # running `ReferenceExtractor` on each of them separately. + # This optimization does not apply to issues from external sources. + def cache_merge_request_closes_issues!(current_user = self.author) + return if project.has_external_issue_tracker? + + transaction do + self.merge_requests_closing_issues.delete_all + + closes_issues(current_user).each do |issue| + self.merge_requests_closing_issues.create!(issue: issue) + end + end + end + def closes_issue?(issue) closes_issues.include?(issue) end @@ -448,7 +547,8 @@ class MergeRequest < ActiveRecord::Base # Return the set of issues that will be closed if this merge request is accepted. def closes_issues(current_user = self.author) if target_branch == project.default_branch - messages = commits.map(&:safe_message) << description + messages = [description] + messages.concat(commits.map(&:safe_message)) if merge_request_diff Gitlab::ClosingIssueExtractor.new(project, current_user). closed_by_message(messages.join("\n")) @@ -514,13 +614,11 @@ class MergeRequest < ActiveRecord::Base end def merge_commit_message - message = "Merge branch '#{source_branch}' into '#{target_branch}'" - message << "\n\n" - message << title.to_s - message << "\n\n" - message << description.to_s - message << "\n\n" - message << "See merge request !#{iid}" + message = "Merge branch '#{source_branch}' into '#{target_branch}'\n\n" + message << "#{title}\n\n" + message << "#{description}\n\n" if description.present? + message << "See merge request #{to_reference}" + message end @@ -564,7 +662,7 @@ class MergeRequest < ActiveRecord::Base end def has_ci? - source_project.ci_service && commits.any? + source_project.try(:ci_service) && commits.any? end def branch_missing? @@ -592,11 +690,14 @@ class MergeRequest < ActiveRecord::Base end def environments - return unless diff_head_commit + return [] unless diff_head_commit - target_project.environments.select do |environment| - environment.includes_commit?(diff_head_commit) - end + @environments ||= + begin + envs = target_project.environments_for(target_branch, diff_head_commit, with_tags: true) + envs.concat(source_project.environments_for(source_branch, diff_head_commit)) if source_project + envs.uniq + end end def state_human_name @@ -674,8 +775,34 @@ class MergeRequest < ActiveRecord::Base diverged_commits_count > 0 end + def commits_sha + commits.map(&:sha) + end + def pipeline - @pipeline ||= source_project.pipeline(diff_head_sha, source_branch) if diff_head_sha && source_project + return unless diff_head_sha && source_project + + @pipeline ||= source_project.pipeline_for(source_branch, diff_head_sha) + end + + def all_pipelines + return unless source_project + + @all_pipelines ||= source_project.pipelines + .where(sha: all_commits_sha, ref: source_branch) + .order(id: :desc) + end + + # Note that this could also return SHA from now dangling commits + # + def all_commits_sha + if persisted? + merge_request_diffs.flat_map(&:commits_sha).uniq + elsif compare_commits + compare_commits.to_a.reverse.map(&:id) + else + [diff_head_sha] + end end def merge_commit @@ -690,12 +817,12 @@ class MergeRequest < ActiveRecord::Base merge_commit end - def support_new_diff_notes? + def has_complete_diff_refs? diff_sha_refs && diff_sha_refs.complete? end def update_diff_notes_positions(old_diff_refs:, new_diff_refs:) - return unless support_new_diff_notes? + return unless has_complete_diff_refs? return if new_diff_refs == old_diff_refs active_diff_notes = self.notes.diff_notes.select do |note| @@ -723,4 +850,30 @@ class MergeRequest < ActiveRecord::Base def keep_around_commit project.repository.keep_around(self.merge_commit_sha) end + + def conflicts + @conflicts ||= Gitlab::Conflict::FileCollection.new(self) + end + + def conflicts_can_be_resolved_by?(user) + access = ::Gitlab::UserAccess.new(user, project: source_project) + access.can_push_to_branch?(source_branch) + end + + def conflicts_can_be_resolved_in_ui? + return @conflicts_can_be_resolved_in_ui if defined?(@conflicts_can_be_resolved_in_ui) + + return @conflicts_can_be_resolved_in_ui = false unless cannot_be_merged? + return @conflicts_can_be_resolved_in_ui = false unless has_complete_diff_refs? + + begin + # Try to parse each conflict. If the MR's mergeable status hasn't been updated, + # ensure that we don't say there are conflicts to resolve when there are no conflict + # files. + conflicts.files.each(&:lines) + @conflicts_can_be_resolved_in_ui = conflicts.files.length > 0 + rescue Rugged::OdbError, Gitlab::Conflict::Parser::UnresolvableError, Gitlab::Conflict::FileCollection::ConflictSideMissing + @conflicts_can_be_resolved_in_ui = false + end + end end diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb new file mode 100644 index 00000000000..99c49a020c9 --- /dev/null +++ b/app/models/merge_request/metrics.rb @@ -0,0 +1,11 @@ +class MergeRequest::Metrics < ActiveRecord::Base + belongs_to :merge_request + + def record! + if merge_request.merged? && self.merged_at.blank? + self.merged_at = Time.now + end + + self.save + end +end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 32cc6a3bfea..dd65a9a8b86 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -6,9 +6,10 @@ class MergeRequestDiff < ActiveRecord::Base # Prevent store of diff if commits amount more then 500 COMMITS_SAFE_SIZE = 100 - belongs_to :merge_request + # Valid types of serialized diffs allowed by Gitlab::Git::Diff + VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta] - delegate :source_branch_sha, :target_branch_sha, :target_branch, :source_branch, to: :merge_request, prefix: nil + belongs_to :merge_request state_machine :state, initial: :empty do state :collected @@ -24,12 +25,51 @@ class MergeRequestDiff < ActiveRecord::Base serialize :st_commits serialize :st_diffs - after_create :reload_content, unless: :importing? - after_save :keep_around_commits, unless: :importing? + # All diff information is collected from repository after object is created. + # It allows you to override variables like head_commit_sha before getting diff. + after_create :save_git_content, unless: :importing? + + def self.select_without_diff + select(column_names - ['st_diffs']) + end + + def st_commits + super || [] + end - def reload_content + # Collect information about commits and diff from repository + # and save it to the database as serialized data + def save_git_content + ensure_commits_sha + save_commits reload_commits - reload_diffs + save_diffs + keep_around_commits + end + + def ensure_commits_sha + merge_request.fetch_ref + self.start_commit_sha ||= merge_request.target_branch_sha + self.head_commit_sha ||= merge_request.source_branch_sha + self.base_commit_sha ||= find_base_sha + save + end + + # Override head_commit_sha to keep compatibility with merge request diff + # created before version 8.4 that does not store head_commit_sha in separate db field. + def head_commit_sha + if persisted? && super.nil? + last_commit.try(:sha) + else + super + end + end + + # This method will rely on repository branch sha + # in case start_commit_sha is nil. Its necesarry for old merge request diff + # created before version 8.4 to work + def safe_start_commit_sha + start_commit_sha || merge_request.target_branch_sha end def size @@ -38,14 +78,11 @@ class MergeRequestDiff < ActiveRecord::Base def raw_diffs(options = {}) if options[:ignore_whitespace_change] - @raw_diffs_no_whitespace ||= begin - compare = Gitlab::Git::Compare.new( + @diffs_no_whitespace ||= + Gitlab::Git::Compare.new( repository.raw_repository, - self.start_commit_sha || self.target_branch_sha, - self.head_commit_sha || self.source_branch_sha, - ) - compare.diffs(options) - end + safe_start_commit_sha, + head_commit_sha).diffs(options) else @raw_diffs ||= {} @raw_diffs[options] ||= load_diffs(st_diffs, options) @@ -53,7 +90,12 @@ class MergeRequestDiff < ActiveRecord::Base end def commits - @commits ||= load_commits(st_commits || []) + @commits ||= load_commits(st_commits) + end + + def reload_commits + @commits = nil + commits end def last_commit @@ -65,53 +107,82 @@ class MergeRequestDiff < ActiveRecord::Base end def base_commit - return unless self.base_commit_sha + return unless base_commit_sha - project.commit(self.base_commit_sha) + project.commit(base_commit_sha) end def start_commit - return unless self.start_commit_sha + return unless start_commit_sha - project.commit(self.start_commit_sha) + project.commit(start_commit_sha) end def head_commit - return last_commit unless self.head_commit_sha + return unless head_commit_sha + + project.commit(head_commit_sha) + end + + def commits_sha + if @commits + commits.map(&:sha) + else + st_commits.map { |commit| commit[:id] } + end + end + + def diff_refs + return unless start_commit_sha || base_commit_sha - project.commit(self.head_commit_sha) + Gitlab::Diff::DiffRefs.new( + base_sha: base_commit_sha, + start_sha: start_commit_sha, + head_sha: head_commit_sha + ) end def diff_refs_by_sha? base_commit_sha? && head_commit_sha? && start_commit_sha? end + def diffs(diff_options = nil) + Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options) + end + + def project + merge_request.target_project + end + def compare @compare ||= - begin - # Update ref for merge request - merge_request.fetch_ref + Gitlab::Git::Compare.new( + repository.raw_repository, + safe_start_commit_sha, + head_commit_sha + ) + end - Gitlab::Git::Compare.new( - repository.raw_repository, - self.target_branch_sha, - self.source_branch_sha - ) - end + def latest? + self == merge_request.merge_request_diff end - private + def compare_with(sha, straight: true) + # When compare merge request versions we want diff A..B instead of A...B + # so we handle cases when user does squash and rebase of the commits between versions. + # For this reason we set straight to true by default. + CompareService.new.execute(project, head_commit_sha, project, sha, straight: straight) + end - # Collect array of Git::Commit objects - # between target and source branches - def unmerged_commits - commits = compare.commits + private - if commits.present? - commits = Commit.decorate(commits, merge_request.source_project).reverse - end + # Old GitLab implementations may have generated diffs as ["--broken-diff"]. + # Avoid an error 500 by ignoring bad elements. See: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776 + def valid_raw_diff?(raw) + return false unless raw.respond_to?(:each) - commits + raw.any? { |element| VALID_CLASSES.include?(element.class) } end def dump_commits(commits) @@ -122,26 +193,21 @@ class MergeRequestDiff < ActiveRecord::Base array.map { |hash| Commit.new(Gitlab::Git::Commit.new(hash), merge_request.source_project) } end - # Reload all commits related to current merge request from repo + # Load all commits related to current merge request diff from repo # and save it as array of hashes in st_commits db field - def reload_commits + def save_commits new_attributes = {} - commit_objects = unmerged_commits + commits = compare.commits - if commit_objects.present? - new_attributes[:st_commits] = dump_commits(commit_objects) + if commits.present? + commits = Commit.decorate(commits, merge_request.source_project).reverse + new_attributes[:st_commits] = dump_commits(commits) end update_columns_serialized(new_attributes) end - # Collect array of Git::Diff objects - # between target and source branches - def unmerged_diffs - compare.diffs(Commit.max_diff_options) - end - def dump_diffs(diffs) if diffs.respond_to?(:map) diffs.map(&:to_hash) @@ -149,7 +215,7 @@ class MergeRequestDiff < ActiveRecord::Base end def load_diffs(raw, options) - if raw.respond_to?(:each) + if valid_raw_diff?(raw) if paths = options[:paths] raw = raw.select do |diff| paths.include?(diff[:old_path]) || paths.include?(diff[:new_path]) @@ -162,16 +228,16 @@ class MergeRequestDiff < ActiveRecord::Base end end - # Reload diffs between branches related to current merge request from repo + # Load diffs between branches related to current merge request diff from repo # and save it as array of hashes in st_diffs db field - def reload_diffs + def save_diffs new_attributes = {} new_diffs = [] if commits.size.zero? new_attributes[:state] = :empty else - diff_collection = unmerged_diffs + diff_collection = compare.diffs(Commit.max_diff_options) if diff_collection.overflow? # Set our state to 'overflow' to make the #empty? and #collected? @@ -188,32 +254,17 @@ class MergeRequestDiff < ActiveRecord::Base end new_attributes[:st_diffs] = new_diffs - - new_attributes[:start_commit_sha] = self.target_branch_sha - new_attributes[:head_commit_sha] = self.source_branch_sha - new_attributes[:base_commit_sha] = branch_base_sha - update_columns_serialized(new_attributes) - - keep_around_commits - end - - def project - merge_request.target_project end def repository project.repository end - def branch_base_commit - return unless self.source_branch_sha && self.target_branch_sha + def find_base_sha + return unless head_commit_sha && start_commit_sha - project.merge_base_commit(self.source_branch_sha, self.target_branch_sha) - end - - def branch_base_sha - branch_base_commit.try(:sha) + project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha) end def utf8_st_diffs @@ -248,8 +299,10 @@ class MergeRequestDiff < ActiveRecord::Base end def keep_around_commits - repository.keep_around(target_branch_sha) - repository.keep_around(source_branch_sha) - repository.keep_around(branch_base_sha) + [repository, merge_request.source_project.repository].each do |repo| + repo.keep_around(start_commit_sha) + repo.keep_around(head_commit_sha) + repo.keep_around(base_commit_sha) + end end end diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb new file mode 100644 index 00000000000..ab597c37947 --- /dev/null +++ b/app/models/merge_requests_closing_issues.rb @@ -0,0 +1,7 @@ +class MergeRequestsClosingIssues < ActiveRecord::Base + belongs_to :merge_request + belongs_to :issue + + validates :merge_request_id, uniqueness: { scope: :issue_id }, presence: true + validates :issue_id, presence: true +end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 2bd7f198030..23aecbfa3a6 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -6,12 +6,16 @@ class Milestone < ActiveRecord::Base Any = MilestoneStruct.new('Any Milestone', '', -1) Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2) + include CacheMarkdownField include InternalId include Sortable include Referable include StripAttribute include Milestoneish + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description + belongs_to :project has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues @@ -158,7 +162,7 @@ class Milestone < ActiveRecord::Base end def title=(value) - write_attribute(:title, Sanitize.clean(value.to_s)) if value.present? + write_attribute(:title, sanitize_title(value)) if value.present? end # Sorts the issues for the given IDs. @@ -204,4 +208,8 @@ class Milestone < ActiveRecord::Base iid end end + + def sanitize_title(value) + CGI.unescape_html(Sanitize.clean(value.to_s)) + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 7c29d27ce97..b67049f0f55 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -1,9 +1,12 @@ class Namespace < ActiveRecord::Base acts_as_paranoid + include CacheMarkdownField include Sortable include Gitlab::ShellAdapter + cache_markdown_field :description, pipeline: :description + has_many :projects, dependent: :destroy belongs_to :owner, class_name: "User" @@ -58,15 +61,13 @@ class Namespace < ActiveRecord::Base def clean_path(path) path = path.dup # Get the email username by removing everything after an `@` sign. - path.gsub!(/@.*\z/, "") - # Usernames can't end in .git, so remove it. - path.gsub!(/\.git\z/, "") - # Remove dashes at the start of the username. - path.gsub!(/\A-+/, "") - # Remove periods at the end of the username. - path.gsub!(/\.+\z/, "") + path.gsub!(/@.*\z/, "") # Remove everything that's not in the list of allowed characters. - path.gsub!(/[^a-zA-Z0-9_\-\.]/, "") + path.gsub!(/[^a-zA-Z0-9_\-\.]/, "") + # Remove trailing violations ('.atom', '.git', or '.') + path.gsub!(/(\.atom|\.git|\.)*\z/, "") + # Remove leading violations ('-') + path.gsub!(/\A\-+/, "") # Users with the great usernames of "." or ".." would end up with a blank username. # Work around that by setting their username to "blank", followed by a counter. @@ -141,6 +142,11 @@ class Namespace < ActiveRecord::Base projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) end + def lfs_enabled? + # User namespace will always default to the global setting + Gitlab.config.lfs.enabled + end + private def repository_storage_paths diff --git a/app/models/note.rb b/app/models/note.rb index ddcd7f9d034..2d644b03e4d 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -6,10 +6,13 @@ class Note < ActiveRecord::Base include Awardable include Importable include FasterCacheKeys + include CacheMarkdownField + + cache_markdown_field :note, pipeline: :note # Attribute containing rendered and redacted Markdown as generated by # Banzai::ObjectRenderer. - attr_accessor :note_html + attr_accessor :redacted_note_html # An Array containing the number of visible references as generated by # Banzai::ObjectRenderer @@ -25,6 +28,9 @@ class Note < ActiveRecord::Base belongs_to :author, class_name: "User" belongs_to :updated_by, class_name: "User" + # Only used by DiffNote, but defined here so that it can be used in `Note.includes` + belongs_to :resolved_by, class_name: "User" + has_many :todos, dependent: :destroy has_many :events, as: :target, dependent: :destroy @@ -59,7 +65,7 @@ class Note < ActiveRecord::Base scope :fresh, ->{ order(created_at: :asc, id: :asc) } scope :inc_author_project, ->{ includes(:project, :author) } scope :inc_author, ->{ includes(:author) } - scope :inc_author_project_award_emoji, ->{ includes(:project, :author, :award_emoji) } + scope :inc_relations_for_view, ->{ includes(:project, :author, :updated_by, :resolved_by, :award_emoji) } scope :diff_notes, ->{ where(type: ['LegacyDiffNote', 'DiffNote']) } scope :non_diff_notes, ->{ where(type: ['Note', nil]) } @@ -70,7 +76,9 @@ class Note < ActiveRecord::Base project: [:project_members, { group: [:group_members] }]) end + after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code + before_validation :set_discussion_id after_save :keep_around_commit class << self @@ -82,13 +90,18 @@ class Note < ActiveRecord::Base [:discussion, noteable_type.try(:underscore), noteable_id].join("-") end + def discussion_id(*args) + Digest::SHA1.hexdigest(build_discussion_id(*args)) + end + def discussions Discussion.for_notes(all) end def grouped_diff_discussions - notes = diff_notes.fresh.select(&:active?) - Discussion.for_diff_notes(notes).map { |d| [d.line_code, d] }.to_h + active_notes = diff_notes.fresh.select(&:active?) + Discussion.for_diff_notes(active_notes). + map { |d| [d.line_code, d] }.to_h end # Searches for notes matching the given query. @@ -129,13 +142,16 @@ class Note < ActiveRecord::Base true end - def discussion_id - @discussion_id ||= - if for_merge_request? - [:discussion, :note, id].join("-") - else - self.class.build_discussion_id(noteable_type, noteable_id || commit_id) - end + def resolvable? + false + end + + def resolved? + false + end + + def to_be_resolved? + resolvable? && !resolved? end def max_attachment_size @@ -243,4 +259,28 @@ class Note < ActiveRecord::Base def nullify_blank_line_code self.line_code = nil if self.line_code.blank? end + + def ensure_discussion_id + return unless self.persisted? + # Needed in case the SELECT statement doesn't ask for `discussion_id` + return unless self.has_attribute?(:discussion_id) + return if self.discussion_id + + set_discussion_id + update_column(:discussion_id, self.discussion_id) + end + + def set_discussion_id + self.discussion_id = Digest::SHA1.hexdigest(build_discussion_id) + end + + def build_discussion_id + if for_merge_request? + # Notes on merge requests are always in a discussion of their own, + # so we generate a unique discussion ID. + [:discussion, :note, SecureRandom.hex].join("-") + else + self.class.build_discussion_id(noteable_type, noteable_id || commit_id) + end + end end diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 121b598b8f3..43fc218de2b 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -32,7 +32,9 @@ class NotificationSetting < ActiveRecord::Base :reopen_merge_request, :close_merge_request, :reassign_merge_request, - :merge_merge_request + :merge_merge_request, + :failed_pipeline, + :success_pipeline ] store :events, accessors: EMAIL_EVENTS, coder: JSON diff --git a/app/models/project.rb b/app/models/project.rb index eefdae35615..bbe590b5a8a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -6,28 +6,38 @@ class Project < ActiveRecord::Base include Gitlab::VisibilityLevel include Gitlab::CurrentSettings include AccessRequestable + include CacheMarkdownField include Referable include Sortable include AfterCommitQueue include CaseSensitivity include TokenAuthenticatable + include ProjectFeaturesCompatibility extend Gitlab::ConfigHelper + class BoardLimitExceeded < StandardError; end + + NUMBER_OF_PERMITTED_BOARDS = 1 UNKNOWN_IMPORT_URL = 'http://unknown.git' + cache_markdown_field :description, pipeline: :description + + delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true + default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level + default_value_for :container_registry_enabled, gitlab_config_features.container_registry + default_value_for(:repository_storage) { current_application_settings.pick_repository_storage } + default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } default_value_for :issues_enabled, gitlab_config_features.issues default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests default_value_for :builds_enabled, gitlab_config_features.builds default_value_for :wiki_enabled, gitlab_config_features.wiki default_value_for :snippets_enabled, gitlab_config_features.snippets - default_value_for :container_registry_enabled, gitlab_config_features.container_registry - default_value_for(:repository_storage) { current_application_settings.repository_storage } - default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } after_create :ensure_dir_exist + after_create :create_project_feature, unless: :project_feature after_save :ensure_dir_exist, if: :namespace_id_changed? # set last_activity_at to the same as created_at @@ -58,11 +68,12 @@ class Project < ActiveRecord::Base alias_attribute :title, :name # Relations - belongs_to :creator, foreign_key: 'creator_id', class_name: 'User' - belongs_to :group, -> { where(type: Group) }, foreign_key: 'namespace_id' + belongs_to :creator, class_name: 'User' + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' belongs_to :namespace - has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event', foreign_key: 'project_id' + has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' + has_many :boards, before_add: :validate_board_limit, dependent: :destroy # Project services has_many :services @@ -70,6 +81,7 @@ class Project < ActiveRecord::Base has_one :drone_ci_service, dependent: :destroy has_one :emails_on_push_service, dependent: :destroy has_one :builds_email_service, dependent: :destroy + has_one :pipelines_email_service, dependent: :destroy has_one :irker_service, dependent: :destroy has_one :pivotaltracker_service, dependent: :destroy has_one :hipchat_service, dependent: :destroy @@ -100,7 +112,7 @@ class Project < ActiveRecord::Base # Merge requests from source project should be kept when source project was removed has_many :fork_merge_requests, foreign_key: 'source_project_id', class_name: MergeRequest has_many :issues, dependent: :destroy - has_many :labels, dependent: :destroy + has_many :labels, dependent: :destroy, class_name: 'ProjectLabel' has_many :services, dependent: :destroy has_many :events, dependent: :destroy has_many :milestones, dependent: :destroy @@ -109,7 +121,7 @@ class Project < ActiveRecord::Base has_many :hooks, dependent: :destroy, class_name: 'ProjectHook' has_many :protected_branches, dependent: :destroy - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'ProjectMember' + has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source alias_method :members, :project_members has_many :users, through: :project_members @@ -128,8 +140,9 @@ class Project < ActiveRecord::Base has_many :notification_settings, dependent: :destroy, as: :source has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" + has_one :project_feature, dependent: :destroy - has_many :commit_statuses, dependent: :destroy, class_name: 'CommitStatus', foreign_key: :gl_project_id + has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id has_many :builds, class_name: 'Ci::Build', foreign_key: :gl_project_id # the builds are created from the commit_statuses has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject', foreign_key: :gl_project_id @@ -140,9 +153,11 @@ class Project < ActiveRecord::Base has_many :deployments, dependent: :destroy accepts_nested_attributes_for :variables, allow_destroy: true + accepts_nested_attributes_for :project_feature delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true + delegate :add_user, to: :team # Validations validates :creator, presence: true, on: :create @@ -157,8 +172,6 @@ class Project < ActiveRecord::Base length: { within: 0..255 }, format: { with: Gitlab::Regex.project_path_regex, message: Gitlab::Regex.project_path_regex_message } - validates :issues_enabled, :merge_requests_enabled, - :wiki_enabled, inclusion: { in: [true, false] } validates :namespace, presence: true validates_uniqueness_of :name, scope: :namespace_id validates_uniqueness_of :path, scope: :namespace_id @@ -194,6 +207,39 @@ class Project < ActiveRecord::Base scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } + scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } + + # "enabled" here means "not disabled". It includes private features! + scope :with_feature_enabled, ->(feature) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => [nil, ProjectFeature::PRIVATE, ProjectFeature::ENABLED] }) + } + + # Picks a feature where the level is exactly that given. + scope :with_feature_access_level, ->(feature, level) { + access_level_attribute = ProjectFeature.access_level_attribute(feature) + with_project_feature.where(project_features: { access_level_attribute => level }) + } + + scope :with_builds_enabled, -> { with_feature_enabled(:builds) } + scope :with_issues_enabled, -> { with_feature_enabled(:issues) } + + # project features may be "disabled", "internal" or "enabled". If "internal", + # they are only available to team members. This scope returns projects where + # the feature is either enabled, or internal with permission for the user. + def self.with_feature_available_for_user(feature, user) + return with_feature_enabled(feature) if user.try(:admin?) + + unconditional = with_feature_access_level(feature, [nil, ProjectFeature::ENABLED]) + return unconditional if user.nil? + + conditional = with_feature_access_level(feature, ProjectFeature::PRIVATE) + authorized = user.authorized_projects.merge(conditional.reorder(nil)) + + union = Gitlab::SQL::Union.new([unconditional.select(:id), authorized.select(:id)]) + where(arel_table[:id].in(Arel::Nodes::SqlLiteral.new(union.to_sql))) + end + scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } @@ -367,18 +413,9 @@ class Project < ActiveRecord::Base %r{(?<project>#{name_pattern}/#{name_pattern})} end - def trending(since = 1.month.ago) - # By counting in the JOIN we don't expose the GROUP BY to the outer query. - # This means that calls such as "any?" and "count" just return a number of - # the total count, instead of the counts grouped per project as a Hash. - join_body = "INNER JOIN ( - SELECT project_id, COUNT(*) AS amount - FROM notes - WHERE created_at >= #{sanitize(since)} - GROUP BY project_id - ) join_note_counts ON projects.id = join_note_counts.project_id" - - joins(join_body).reorder('join_note_counts.amount DESC') + def trending + joins('INNER JOIN trending_projects ON projects.id = trending_projects.project_id'). + reorder('trending_projects.id ASC') end def cached_count @@ -386,6 +423,16 @@ class Project < ActiveRecord::Base Project.count end end + + def group_ids + joins(:namespace).where(namespaces: { type: 'Group' }).select(:namespace_id) + end + end + + def lfs_enabled? + return namespace.lfs_enabled? if self[:lfs_enabled].nil? + + self[:lfs_enabled] && Gitlab.config.lfs.enabled end def repository_storage_path @@ -434,7 +481,7 @@ class Project < ActiveRecord::Base # ref can't be HEAD, can only be branch/tag name or SHA def latest_successful_builds_for(ref = default_branch) - latest_pipeline = pipelines.latest_successful_for(ref).first + latest_pipeline = pipelines.latest_successful_for(ref) if latest_pipeline latest_pipeline.builds.latest.with_artifacts @@ -469,8 +516,6 @@ class Project < ActiveRecord::Base end def reset_cache_and_import_attrs - update(import_error: nil) - ProjectCacheWorker.perform_async(self.id) self.import_data.destroy if self.import_data @@ -485,7 +530,7 @@ class Project < ActiveRecord::Base end def import_url - if import_data && super + if import_data && super.present? import_url = Gitlab::UrlSanitizer.new(super, credentials: import_data.credentials) import_url.full_url else @@ -609,10 +654,12 @@ class Project < ActiveRecord::Base end def new_issue_address(author) - if Gitlab::IncomingEmail.enabled? && author - Gitlab::IncomingEmail.reply_address( - "#{path_with_namespace}+#{author.authentication_token}") - end + return unless Gitlab::IncomingEmail.supports_issue_creation? && author + + author.ensure_incoming_email_token! + + Gitlab::IncomingEmail.reply_address( + "#{path_with_namespace}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -655,6 +702,10 @@ class Project < ActiveRecord::Base end end + def issue_reference_pattern + issues_tracker.reference_pattern + end + def default_issues_tracker? !external_issue_tracker end @@ -677,6 +728,10 @@ class Project < ActiveRecord::Base update_column(:has_external_issue_tracker, services.external_issue_trackers.any?) end + def has_wiki? + wiki_enabled? || has_external_wiki? + end + def external_wiki if has_external_wiki.nil? cache_has_external_wiki # Populate @@ -706,7 +761,7 @@ class Project < ActiveRecord::Base if template.nil? # If no template, we should create an instance. Ex `create_gitlab_ci_service` - self.send :"create_#{service_name}_service" + public_send("create_#{service_name}_service") else Service.create_from_template(self.id, template) end @@ -716,10 +771,8 @@ class Project < ActiveRecord::Base def create_labels Label.templates.each do |label| - label = label.dup - label.template = nil - label.project_id = self.id - label.save + params = label.attributes.except('id', 'template', 'created_at', 'updated_at') + Labels::FindOrCreateService.new(nil, self, params).execute(skip_authorization: true) end end @@ -817,11 +870,6 @@ class Project < ActiveRecord::Base end end - def update_merge_requests(oldrev, newrev, ref, user) - MergeRequests::RefreshService.new(self, user). - execute(oldrev, newrev, ref) - end - def valid_repo? repository.exists? rescue @@ -1001,10 +1049,6 @@ class Project < ActiveRecord::Base project_members.find_by(user_id: user) end - def add_user(user, access_level, current_user = nil) - team.add_user(user, access_level, current_user) - end - def default_branch @default_branch ||= repository.root_ref if repository.exists? end @@ -1032,6 +1076,7 @@ class Project < ActiveRecord::Base "refs/heads/#{branch}", force: true) repository.copy_gitattributes(branch) + repository.expire_avatar_cache(branch) reload_default_branch end @@ -1051,10 +1096,6 @@ class Project < ActiveRecord::Base forks.count end - def find_label(name) - labels.find_by(name: name) - end - def origin_merge_requests merge_requests.where(source_project_id: self.id) end @@ -1092,16 +1133,21 @@ class Project < ActiveRecord::Base !namespace.share_with_group_lock end - def pipeline(sha, ref) + def pipeline_for(ref, sha = nil) + sha ||= commit(ref).try(:sha) + + return unless sha + pipelines.order(id: :desc).find_by(sha: sha, ref: ref) end - def ensure_pipeline(sha, ref, current_user = nil) - pipeline(sha, ref) || pipelines.create(sha: sha, ref: ref, user: current_user) + def ensure_pipeline(ref, sha, current_user = nil) + pipeline_for(ref, sha) || + pipelines.create(sha: sha, ref: ref, user: current_user) end def enable_ci - self.builds_enabled = true + project_feature.update_attribute(:builds_access_level, ProjectFeature::ENABLED) end def any_runners?(&block) @@ -1116,12 +1162,6 @@ class Project < ActiveRecord::Base self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - # TODO (ayufan): For now we use runners_token (backward compatibility) - # In 8.4 every build will have its own individual token valid for time of build - def valid_build_token?(token) - self.builds_enabled? && self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) - end - def build_coverage_enabled? build_coverage_regex.present? end @@ -1266,8 +1306,40 @@ class Project < ActiveRecord::Base end end + def pushes_since_gc + Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i } + end + + def increment_pushes_since_gc + Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) } + end + + def reset_pushes_since_gc + Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } + end + + def environments_for(ref, commit, with_tags: false) + environment_ids = deployments.group(:environment_id). + select(:environment_id) + + environment_ids = + if with_tags + environment_ids.where('ref=? OR tag IS TRUE', ref) + else + environment_ids.where(ref: ref) + end + + environments.available.where(id: environment_ids).select do |environment| + environment.includes_commit?(commit) + end + end + private + def pushes_since_gc_redis_key + "projects/#{id}/pushes_since_gc" + end + def default_branch_protected? current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE @@ -1296,4 +1368,15 @@ class Project < ActiveRecord::Base shared_projects.any? end + + # Similar to the normal callbacks that hook into the life cycle of an + # Active Record object, you can also define callbacks that get triggered + # when you add an object to an association collection. If any of these + # callbacks throw an exception, the object will not be added to the + # collection. Before you add a new board to the boards collection if you + # already have 1, 2, or n it will fail, but it if you have 0 that is lower + # than the number of permitted boards per project it won't fail. + def validate_board_limit(board) + raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS + end end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb new file mode 100644 index 00000000000..34fd5a57b5e --- /dev/null +++ b/app/models/project_feature.rb @@ -0,0 +1,95 @@ +class ProjectFeature < ActiveRecord::Base + # == Project features permissions + # + # Grants access level to project tools + # + # Tools can be enabled only for users, everyone or disabled + # Access control is made only for non private projects + # + # levels: + # + # Disabled: not enabled for anyone + # Private: enabled only for team members + # Enabled: enabled for everyone able to access the project + # + + # Permission levels + DISABLED = 0 + PRIVATE = 10 + ENABLED = 20 + + FEATURES = %i(issues merge_requests wiki snippets builds repository) + + class << self + def access_level_attribute(feature) + feature = feature.model_name.plural.to_sym if feature.respond_to?(:model_name) + raise ArgumentError, "invalid project feature: #{feature}" unless FEATURES.include?(feature) + + "#{feature}_access_level".to_sym + end + end + + # Default scopes force us to unscope here since a service may need to check + # permissions for a project in pending_delete + # http://stackoverflow.com/questions/1540645/how-to-disable-default-scope-for-a-belongs-to + belongs_to :project, -> { unscope(where: :pending_delete) } + + validate :repository_children_level + + default_value_for :builds_access_level, value: ENABLED, allows_nil: false + default_value_for :issues_access_level, value: ENABLED, allows_nil: false + default_value_for :merge_requests_access_level, value: ENABLED, allows_nil: false + default_value_for :snippets_access_level, value: ENABLED, allows_nil: false + default_value_for :wiki_access_level, value: ENABLED, allows_nil: false + default_value_for :repository_access_level, value: ENABLED, allows_nil: false + + def feature_available?(feature, user) + access_level = public_send(ProjectFeature.access_level_attribute(feature)) + get_permission(user, access_level) + end + + def builds_enabled? + return true unless builds_access_level + + builds_access_level > DISABLED + end + + def wiki_enabled? + return true unless wiki_access_level + + wiki_access_level > DISABLED + end + + def merge_requests_enabled? + return true unless merge_requests_access_level + + merge_requests_access_level > DISABLED + end + + private + + # Validates builds and merge requests access level + # which cannot be higher than repository access level + def repository_children_level + validator = lambda do |field| + level = public_send(field) || ProjectFeature::ENABLED + not_allowed = level > repository_access_level + self.errors.add(field, "cannot have higher visibility level than repository access level") if not_allowed + end + + %i(merge_requests_access_level builds_access_level).each(&validator) + end + + def get_permission(user, level) + case level + when DISABLED + false + when PRIVATE + user && (project.team.member?(user) || user.admin?) + when ENABLED + true + else + true + end + end +end diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index e52a6bd7c84..db46def11eb 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -1,4 +1,6 @@ class ProjectGroupLink < ActiveRecord::Base + include Expirable + GUEST = 10 REPORTER = 20 DEVELOPER = 30 @@ -8,7 +10,7 @@ class ProjectGroupLink < ActiveRecord::Base belongs_to :group validates :project_id, presence: true - validates :group_id, presence: true + validates :group, presence: true validates :group_id, uniqueness: { scope: [:project_id], message: "already shared with this group" } validates :group_access, presence: true validates :group_access, inclusion: { in: Gitlab::Access.values }, presence: true @@ -26,7 +28,7 @@ class ProjectGroupLink < ActiveRecord::Base self.class.access_options.key(self.group_access) end - private + private def different_group if self.group && self.project && self.project.group == self.group diff --git a/app/models/project_label.rb b/app/models/project_label.rb new file mode 100644 index 00000000000..82f47f0e8fd --- /dev/null +++ b/app/models/project_label.rb @@ -0,0 +1,38 @@ +class ProjectLabel < Label + MAX_NUMBER_OF_PRIORITIES = 1 + + belongs_to :project + + validates :project, presence: true + + validate :permitted_numbers_of_priorities + validate :title_must_not_exist_at_group_level + + delegate :group, to: :project, allow_nil: true + + alias_attribute :subject, :project + + def subject_foreign_key + 'project_id' + end + + def to_reference(target_project = nil, format: :id) + super(project, target_project, format: format) + end + + private + + def title_must_not_exist_at_group_level + return unless group.present? && title_changed? + + if group.labels.with_title(self.title).exists? + errors.add(:title, :label_already_exists_at_group_level, group: group.name) + end + end + + def permitted_numbers_of_priorities + if priorities && priorities.size > MAX_NUMBER_OF_PRIORITIES + errors.add(:priorities, 'Number of permitted priorities exceeded') + end + end +end diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 81af55aa29a..338e685339a 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -1,4 +1,6 @@ class BugzillaService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title diff --git a/app/models/project_services/builds_email_service.rb b/app/models/project_services/builds_email_service.rb index fa66e5864b8..201b94b065b 100644 --- a/app/models/project_services/builds_email_service.rb +++ b/app/models/project_services/builds_email_service.rb @@ -43,7 +43,7 @@ class BuildsEmailService < Service end def can_test? - project.builds.count > 0 + project.builds.any? end def disabled_title diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index 63a5ed14484..b2f426dc2ac 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -1,4 +1,6 @@ class CustomIssueTrackerService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title @@ -9,6 +11,10 @@ class CustomIssueTrackerService < IssueTrackerService end end + def title=(value) + self.properties['title'] = value if self.properties + end + def description if self.properties && self.properties['description'].present? self.properties['description'] diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index 5d17c358330..6bd8d4ec568 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -1,6 +1,8 @@ class GitlabIssueTrackerService < IssueTrackerService include Gitlab::Routing.url_helpers + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url default_value_for :default, true diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index d7c986c1a91..660a8ae3421 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -1,5 +1,12 @@ class HipchatService < Service + include ActionView::Helpers::SanitizeHelper + MAX_COMMITS = 3 + HIPCHAT_ALLOWED_TAGS = %w[ + a b i strong em br img pre code + table th tr td caption colgroup col thead tbody tfoot + ul ol li dl dt dd + ] prop_accessor :token, :room, :server, :notify, :color, :api_version boolean_accessor :notify_only_broken_builds @@ -39,7 +46,7 @@ class HipchatService < Service end def supported_events - %w(push issue merge_request note tag_push build) + %w(push issue confidential_issue merge_request note tag_push build) end def execute(data) @@ -88,6 +95,10 @@ class HipchatService < Service end end + def render_line(text) + markdown(text.lines.first.chomp, pipeline: :single_line) if text + end + def create_push_message(push) ref_type = Gitlab::Git.tag_ref?(push[:ref]) ? 'tag' : 'branch' ref = Gitlab::Git.ref_name(push[:ref]) @@ -110,7 +121,7 @@ class HipchatService < Service message << "(<a href=\"#{project.web_url}/compare/#{before}...#{after}\">Compare changes</a>)" push[:commits].take(MAX_COMMITS).each do |commit| - message << "<br /> - #{commit[:message].lines.first} (<a href=\"#{commit[:url]}\">#{commit[:id][0..5]}</a>)" + message << "<br /> - #{render_line(commit[:message])} (<a href=\"#{commit[:url]}\">#{commit[:id][0..5]}</a>)" end if push[:commits].count > MAX_COMMITS @@ -121,12 +132,22 @@ class HipchatService < Service message end - def format_body(body) - if body - body = body.truncate(200, separator: ' ', omission: '...') - end + def markdown(text, options = {}) + return "" unless text + + context = { + project: project, + pipeline: :email + } + + Banzai.render(text, context) - "<pre>#{body}</pre>" + context.merge!(options) + + html = Banzai.post_process(Banzai.render(text, context), context) + sanitized_html = sanitize(html, tags: HIPCHAT_ALLOWED_TAGS, attributes: %w[href title alt]) + + sanitized_html.truncate(200, separator: ' ', omission: '...') end def create_issue_message(data) @@ -134,7 +155,7 @@ class HipchatService < Service obj_attr = data[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) - title = obj_attr[:title] + title = render_line(obj_attr[:title]) state = obj_attr[:state] issue_iid = obj_attr[:iid] issue_url = obj_attr[:url] @@ -143,10 +164,7 @@ class HipchatService < Service issue_link = "<a href=\"#{issue_url}\">issue ##{issue_iid}</a>" message = "#{user_name} #{state} #{issue_link} in #{project_link}: <b>#{title}</b>" - if description - description = format_body(description) - message << description - end + message << "<pre>#{markdown(description)}</pre>" message end @@ -159,23 +177,20 @@ class HipchatService < Service merge_request_id = obj_attr[:iid] state = obj_attr[:state] description = obj_attr[:description] - title = obj_attr[:title] + title = render_line(obj_attr[:title]) merge_request_url = "#{project_url}/merge_requests/#{merge_request_id}" merge_request_link = "<a href=\"#{merge_request_url}\">merge request !#{merge_request_id}</a>" message = "#{user_name} #{state} #{merge_request_link} in " \ "#{project_link}: <b>#{title}</b>" - if description - description = format_body(description) - message << description - end + message << "<pre>#{markdown(description)}</pre>" message end def format_title(title) - "<b>" + title.lines.first.chomp + "</b>" + "<b>#{render_line(title)}</b>" end def create_note_message(data) @@ -186,11 +201,13 @@ class HipchatService < Service note = obj_attr[:note] note_url = obj_attr[:url] noteable_type = obj_attr[:noteable_type] + commit_id = nil case noteable_type when "Commit" commit_attr = HashWithIndifferentAccess.new(data[:commit]) - subject_desc = commit_attr[:id] + commit_id = commit_attr[:id] + subject_desc = commit_id subject_desc = Commit.truncate_sha(subject_desc) subject_type = "commit" title = format_title(commit_attr[:message]) @@ -218,10 +235,7 @@ class HipchatService < Service message = "#{user_name} commented on #{subject_html} in #{project_link}: " message << title - if note - note = format_body(note) - message << note - end + message << "<pre>#{markdown(note, ref: commit_id)}</pre>" message end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index d1df6d0292f..207bb816ad1 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -1,8 +1,12 @@ class IssueTrackerService < Service - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? - default_value_for :category, 'issue_tracker' + # Pattern used to extract links from comments + # Override this method on services that uses different patterns + def reference_pattern + @reference_pattern ||= %r{(\b[A-Z][A-Z0-9_]+-|#{Issue.reference_prefix})(?<issue>\d+)} + end + def default? default end @@ -32,18 +36,24 @@ class IssueTrackerService < Service ] end - def initialize_properties - if properties.nil? - if enabled_in_gitlab_config + # Initialize with default properties values + # or receive a block with custom properties + def initialize_properties(&block) + return unless properties.nil? + + if enabled_in_gitlab_config + if block_given? + yield + else self.properties = { title: issues_tracker['title'], project_url: issues_tracker['project_url'], issues_url: issues_tracker['issues_url'], new_issue_url: issues_tracker['new_issue_url'] } - else - self.properties = {} end + else + self.properties = {} end end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 97bcbacf2b9..2dbe0075465 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -1,28 +1,80 @@ +# == Schema Information +# +# Table name: services +# +# id :integer not null, primary key +# type :string(255) +# title :string(255) +# project_id :integer +# created_at :datetime +# updated_at :datetime +# active :boolean default(FALSE), not null +# properties :text +# template :boolean default(FALSE) +# push_events :boolean default(TRUE) +# issues_events :boolean default(TRUE) +# merge_requests_events :boolean default(TRUE) +# tag_push_events :boolean default(TRUE) +# note_events :boolean default(TRUE), not null +# build_events :boolean default(FALSE), not null +# + class JiraService < IssueTrackerService - include HTTParty include Gitlab::Routing.url_helpers - DEFAULT_API_VERSION = 2 + validates :url, url: true, presence: true, if: :activated? + validates :project_key, presence: true, if: :activated? - prop_accessor :username, :password, :api_url, :jira_issue_transition_id, - :title, :description, :project_url, :issues_url, :new_issue_url + prop_accessor :username, :password, :url, :project_key, + :jira_issue_transition_id, :title, :description - validates :api_url, presence: true, url: true, if: :activated? + before_update :reset_password - before_validation :set_api_url, :set_jira_issue_transition_id + # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 + def reference_pattern + @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} + end - before_update :reset_password + def initialize_properties + super do + self.properties = { + title: issues_tracker['title'], + url: issues_tracker['url'] + } + end + end def reset_password # don't reset the password if a new one is provided - if api_url_changed? && !password_touched? + if url_changed? && !password_touched? self.password = nil end end + def options + url = URI.parse(self.url) + + { + username: self.username, + password: self.password, + site: URI.join(url, '/').to_s, + context_path: url.path, + auth_type: :basic, + read_timeout: 120, + use_ssl: url.scheme == 'https' + } + end + + def client + @client ||= JIRA::Client.new(options) + end + + def jira_project + @jira_project ||= client.Project.find(project_key) + end + def help - 'Setting `project_url`, `issues_url` and `new_issue_url` will '\ - 'allow a user to easily navigate to the Jira issue tracker. See the '\ + 'See the ' \ '[integration doc](http://doc.gitlab.com/ce/integration/external-issue-tracker.html) '\ 'for details.' end @@ -48,12 +100,26 @@ class JiraService < IssueTrackerService end def fields - super.push( - { type: 'text', name: 'api_url', placeholder: 'https://jira.example.com/rest/api/2' }, + [ + { type: 'text', name: 'url', title: 'URL', placeholder: 'https://jira.example.com' }, + { type: 'text', name: 'project_key', placeholder: 'Project Key' }, { type: 'text', name: 'username', placeholder: '' }, { type: 'password', name: 'password', placeholder: '' }, { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' } - ) + ] + end + + # URLs to redirect from Gitlab issues pages to jira issue tracker + def project_url + "#{url}/issues/?jql=project=#{project_key}" + end + + def issues_url + "#{url}/browse/:id" + end + + def new_issue_url + "#{url}/secure/CreateIssue.jspa" end def execute(push, issue = nil) @@ -67,7 +133,7 @@ class JiraService < IssueTrackerService end def create_cross_reference_note(mentioned, noteable, author) - issue_name = mentioned.id + issue_key = mentioned.id project = self.project noteable_name = noteable.class.name.underscore.downcase noteable_id = if noteable.is_a?(Commit) @@ -94,58 +160,43 @@ class JiraService < IssueTrackerService } } - add_comment(data, issue_name) + add_comment(data, issue_key) end - def test_settings - return unless api_url.present? - result = JiraService.get( - jira_api_test_url, - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) - - case result.code - when 201, 200 - Rails.logger.info("#{self.class.name} SUCCESS #{result.code}: Successfully connected to #{api_url}.") - true - else - Rails.logger.info("#{self.class.name} ERROR #{result.code}: #{result.parsed_response}") - false - end - rescue Errno::ECONNREFUSED => e - Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{api_url}." - false + # reason why service cannot be tested + def disabled_title + "Please fill in Password and Username." end - private - - def build_api_url_from_project_url - server = URI(project_url) - default_ports = [["http", 80], ["https", 443]].include?([server.scheme, server.port]) - server_url = "#{server.scheme}://#{server.host}" - server_url.concat(":#{server.port}") unless default_ports - "#{server_url}/rest/api/#{DEFAULT_API_VERSION}" - rescue - "" # looks like project URL was not valid + def can_test? + username.present? && password.present? end - def set_api_url - self.api_url = build_api_url_from_project_url if self.api_url.blank? + # JIRA does not need test data. + # We are requesting the project that belongs to the project key. + def test_data(user = nil, project = nil) + nil end - def set_jira_issue_transition_id - self.jira_issue_transition_id ||= "2" + def test_settings + return unless url.present? + # Test settings by getting the project + jira_project + + rescue Errno::ECONNREFUSED, JIRA::HTTPError => e + Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{url}." + false end + private + def close_issue(entity, issue) commit_id = if entity.is_a?(Commit) entity.id elsif entity.is_a?(MergeRequest) entity.diff_head_sha end + commit_url = build_entity_url(:commit, commit_id) # Depending on the JIRA project's workflow, a comment during transition @@ -156,24 +207,16 @@ class JiraService < IssueTrackerService end def transition_issue(issue) - message = { - transition: { - id: jira_issue_transition_id - } - } - send_message(close_issue_url(issue.iid), message.to_json) + issue = client.Issue.find(issue.iid) + issue.transitions.build.save(transition: { id: jira_issue_transition_id }) end def add_issue_solved_comment(issue, commit_id, commit_url) - comment = { - body: "Issue solved with [#{commit_id}|#{commit_url}]." - } - - send_message(comment_url(issue.iid), comment.to_json) + comment = "Issue solved with [#{commit_id}|#{commit_url}]." + send_message(issue.iid, comment) end - def add_comment(data, issue_name) - url = comment_url(issue_name) + def add_comment(data, issue_key) user_name = data[:user][:name] user_url = data[:user][:url] entity_name = data[:entity][:name] @@ -181,72 +224,35 @@ class JiraService < IssueTrackerService entity_title = data[:entity][:title] project_name = data[:project][:name] - message = { - body: %Q{[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'} - } + message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'" - unless existing_comment?(issue_name, message[:body]) - send_message(url, message.to_json) + unless comment_exists?(issue_key, message) + send_message(issue_key, message) end end - def auth - require 'base64' - Base64.urlsafe_encode64("#{self.username}:#{self.password}") - end - - def send_message(url, message) - return unless api_url.present? - result = JiraService.post( - url, - body: message, - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) - - message = case result.code - when 201, 200, 204 - "#{self.class.name} SUCCESS #{result.code}: Successfully posted to #{url}." - when 401 - "#{self.class.name} ERROR 401: Unauthorized. Check the #{self.username} credentials and JIRA access permissions and try again." - else - "#{self.class.name} ERROR #{result.code}: #{result.parsed_response}" - end - - Rails.logger.info(message) - message - rescue URI::InvalidURIError, Errno::ECONNREFUSED => e - Rails.logger.info "#{self.class.name} ERROR: #{e.message}. Hostname: #{url}." + def comment_exists?(issue_key, message) + comments = client.Issue.find(issue_key).comments + comments.map { |comment| comment.body.include?(message) }.any? end - def existing_comment?(issue_name, new_comment) - return unless api_url.present? - result = JiraService.get( - comment_url(issue_name), - headers: { - 'Content-Type' => 'application/json', - 'Authorization' => "Basic #{auth}" - } - ) + def send_message(issue_key, message) + return unless url.present? - case result.code - when 201, 200 - existing_comments = JSON.parse(result.body)['comments'] + issue = client.Issue.find(issue_key) - if existing_comments.present? - return existing_comments.map { |comment| comment['body'].include?(new_comment) }.any? - end + if issue.comments.build.save!(body: message) + result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}." end - false - rescue JSON::ParserError - false + Rails.logger.info(result_message) + result_message + rescue URI::InvalidURIError, Errno::ECONNREFUSED, JIRA::HTTPError => e + Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}" end def resource_url(resource) - "#{Settings.gitlab['url'].chomp("/")}#{resource}" + "#{Settings.gitlab.base_url.chomp("/")}#{resource}" end def build_entity_url(entity_name, entity_id) @@ -262,16 +268,4 @@ class JiraService < IssueTrackerService ) ) end - - def close_issue_url(issue_name) - "#{self.api_url}/issue/#{issue_name}/transitions" - end - - def comment_url(issue_name) - "#{self.api_url}/issue/#{issue_name}/comment" - end - - def jira_api_test_url - "#{self.api_url}/myself" - end end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb new file mode 100644 index 00000000000..745f9bd1b43 --- /dev/null +++ b/app/models/project_services/pipelines_email_service.rb @@ -0,0 +1,84 @@ +class PipelinesEmailService < Service + prop_accessor :recipients + boolean_accessor :notify_only_broken_pipelines + validates :recipients, presence: true, if: :activated? + + def initialize_properties + self.properties ||= { notify_only_broken_pipelines: true } + end + + def title + 'Pipelines emails' + end + + def description + 'Email the pipelines status to a list of recipients.' + end + + def to_param + 'pipelines_email' + end + + def supported_events + %w[pipeline] + end + + def execute(data, force: false) + return unless supported_events.include?(data[:object_kind]) + return unless force || should_pipeline_be_notified?(data) + + all_recipients = retrieve_recipients(data) + + return unless all_recipients.any? + + pipeline_id = data[:object_attributes][:id] + PipelineNotificationWorker.new.perform(pipeline_id, all_recipients) + end + + def can_test? + project.pipelines.any? + end + + def disabled_title + 'Please setup a pipeline on your repository.' + end + + def test_data(project, user) + data = Gitlab::DataBuilder::Pipeline.build(project.pipelines.last) + data[:user] = user.hook_attrs + data + end + + def fields + [ + { type: 'textarea', + name: 'recipients', + placeholder: 'Emails separated by comma' }, + { type: 'checkbox', + name: 'notify_only_broken_pipelines' }, + ] + end + + def test(data) + result = execute(data, force: true) + + { success: true, result: result } + rescue StandardError => error + { success: false, result: error } + end + + def should_pipeline_be_notified?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end + + def retrieve_recipients(data) + recipients.to_s.split(',').reject(&:blank?) + end +end diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index f634e0772c0..f9da273cf08 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -1,4 +1,6 @@ class RedmineService < IssueTrackerService + validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url def title diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb index abbc780dc1a..e1b937817f4 100644 --- a/app/models/project_services/slack_service.rb +++ b/app/models/project_services/slack_service.rb @@ -1,6 +1,6 @@ class SlackService < Service prop_accessor :webhook, :username, :channel - boolean_accessor :notify_only_broken_builds + boolean_accessor :notify_only_broken_builds, :notify_only_broken_pipelines validates :webhook, presence: true, url: true, if: :activated? def initialize_properties @@ -10,6 +10,7 @@ class SlackService < Service if properties.nil? self.properties = {} self.notify_only_broken_builds = true + self.notify_only_broken_pipelines = true end end @@ -38,13 +39,15 @@ class SlackService < Service { type: 'text', name: 'username', placeholder: 'username' }, { type: 'text', name: 'channel', placeholder: "#general" }, { type: 'checkbox', name: 'notify_only_broken_builds' }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, ] default_fields + build_event_channels end def supported_events - %w(push issue merge_request note tag_push build wiki_page) + %w[push issue confidential_issue merge_request note tag_push + build pipeline wiki_page] end def execute(data) @@ -62,32 +65,22 @@ class SlackService < Service # 'close' action. Ignore update events for now to prevent duplicate # messages from arriving. - message = \ - case object_kind - when "push", "tag_push" - PushMessage.new(data) - when "issue" - IssueMessage.new(data) unless is_update?(data) - when "merge_request" - MergeMessage.new(data) unless is_update?(data) - when "note" - NoteMessage.new(data) - when "build" - BuildMessage.new(data) if should_build_be_notified?(data) - when "wiki_page" - WikiPageMessage.new(data) - end - - opt = {} - - event_channel = get_channel_field(object_kind) || channel - - opt[:channel] = event_channel if event_channel - opt[:username] = username if username + message = get_message(object_kind, data) if message + opt = {} + + event_channel = get_channel_field(object_kind) || channel + + opt[:channel] = event_channel if event_channel + opt[:username] = username if username + notifier = Slack::Notifier.new(webhook, opt) notifier.ping(message.pretext, attachments: message.attachments, fallback: message.fallback) + + true + else + false end end @@ -105,6 +98,25 @@ class SlackService < Service private + def get_message(object_kind, data) + case object_kind + when "push", "tag_push" + PushMessage.new(data) + when "issue" + IssueMessage.new(data) unless is_update?(data) + when "merge_request" + MergeMessage.new(data) unless is_update?(data) + when "note" + NoteMessage.new(data) + when "build" + BuildMessage.new(data) if should_build_be_notified?(data) + when "pipeline" + PipelineMessage.new(data) if should_pipeline_be_notified?(data) + when "wiki_page" + WikiPageMessage.new(data) + end + end + def get_channel_field(event) field_name = event_channel_name(event) self.public_send(field_name) @@ -142,6 +154,17 @@ class SlackService < Service false end end + + def should_pipeline_be_notified?(data) + case data[:object_attributes][:status] + when 'success' + !notify_only_broken_pipelines? + when 'failed' + true + else + false + end + end end require "slack_service/issue_message" @@ -149,4 +172,5 @@ require "slack_service/push_message" require "slack_service/merge_message" require "slack_service/note_message" require "slack_service/build_message" +require "slack_service/pipeline_message" require "slack_service/wiki_page_message" diff --git a/app/models/project_services/slack_service/build_message.rb b/app/models/project_services/slack_service/build_message.rb index 69c21b3fc38..0fca4267bad 100644 --- a/app/models/project_services/slack_service/build_message.rb +++ b/app/models/project_services/slack_service/build_message.rb @@ -9,7 +9,7 @@ class SlackService attr_reader :user_name attr_reader :duration - def initialize(params, commit = true) + def initialize(params) @sha = params[:sha] @ref_type = params[:tag] ? 'tag' : 'branch' @ref = params[:ref] @@ -36,7 +36,7 @@ class SlackService def message "#{project_link}: Commit #{commit_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}" - end + end def format(string) Slack::Notifier::LinkFormatter.format(string) diff --git a/app/models/project_services/slack_service/issue_message.rb b/app/models/project_services/slack_service/issue_message.rb index 88e053ec192..cd87a79d0c6 100644 --- a/app/models/project_services/slack_service/issue_message.rb +++ b/app/models/project_services/slack_service/issue_message.rb @@ -11,7 +11,7 @@ class SlackService attr_reader :description def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_services/slack_service/merge_message.rb b/app/models/project_services/slack_service/merge_message.rb index 11fc691022b..b7615c96068 100644 --- a/app/models/project_services/slack_service/merge_message.rb +++ b/app/models/project_services/slack_service/merge_message.rb @@ -10,7 +10,7 @@ class SlackService attr_reader :title def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_services/slack_service/note_message.rb b/app/models/project_services/slack_service/note_message.rb index 89ba51cb662..9e84e90f38c 100644 --- a/app/models/project_services/slack_service/note_message.rb +++ b/app/models/project_services/slack_service/note_message.rb @@ -10,7 +10,7 @@ class SlackService def initialize(params) params = HashWithIndifferentAccess.new(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_services/slack_service/pipeline_message.rb b/app/models/project_services/slack_service/pipeline_message.rb new file mode 100644 index 00000000000..f06b3562965 --- /dev/null +++ b/app/models/project_services/slack_service/pipeline_message.rb @@ -0,0 +1,79 @@ +class SlackService + class PipelineMessage < BaseMessage + attr_reader :sha, :ref_type, :ref, :status, :project_name, :project_url, + :user_name, :duration, :pipeline_id + + def initialize(data) + pipeline_attributes = data[:object_attributes] + @sha = pipeline_attributes[:sha] + @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' + @ref = pipeline_attributes[:ref] + @status = pipeline_attributes[:status] + @duration = pipeline_attributes[:duration] + @pipeline_id = pipeline_attributes[:id] + + @project_name = data[:project][:path_with_namespace] + @project_url = data[:project][:web_url] + @user_name = data[:commit] && data[:commit][:author_name] + end + + def pretext + '' + end + + def fallback + format(message) + end + + def attachments + [{ text: format(message), color: attachment_color }] + end + + private + + def message + "#{project_link}: Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}" + end + + def format(string) + Slack::Notifier::LinkFormatter.format(string) + end + + def humanized_status + case status + when 'success' + 'passed' + else + status + end + end + + def attachment_color + if status == 'success' + 'good' + else + 'danger' + end + end + + def branch_url + "#{project_url}/commits/#{ref}" + end + + def branch_link + "[#{ref}](#{branch_url})" + end + + def project_link + "[#{project_name}](#{project_url})" + end + + def pipeline_url + "#{project_url}/pipelines/#{pipeline_id}" + end + + def pipeline_link + "[#{Commit.truncate_sha(sha)}](#{pipeline_url})" + end + end +end diff --git a/app/models/project_services/slack_service/wiki_page_message.rb b/app/models/project_services/slack_service/wiki_page_message.rb index f336d9e7691..160ca3ac115 100644 --- a/app/models/project_services/slack_service/wiki_page_message.rb +++ b/app/models/project_services/slack_service/wiki_page_message.rb @@ -9,7 +9,7 @@ class SlackService attr_reader :description def initialize(params) - @user_name = params[:user][:name] + @user_name = params[:user][:username] @project_name = params[:project_name] @project_url = params[:project_url] diff --git a/app/models/project_team.rb b/app/models/project_team.rb index d0a714cd6fc..a6e911df9bd 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -15,9 +15,9 @@ class ProjectTeam users, access, current_user = *args if users.respond_to?(:each) - add_users(users, access, current_user) + add_users(users, access, current_user: current_user) else - add_user(users, access, current_user) + add_user(users, access, current_user: current_user) end end @@ -33,17 +33,24 @@ class ProjectTeam member end - def add_users(users, access, current_user = nil) + def add_users(users, access_level, current_user: nil, expires_at: nil) ProjectMember.add_users_to_projects( [project.id], users, - access, - current_user + access_level, + current_user: current_user, + expires_at: expires_at ) end - def add_user(user, access, current_user = nil) - add_users([user], access, current_user) + def add_user(user, access_level, current_user: nil, expires_at: nil) + ProjectMember.add_user( + project, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) end # Remove all users from project team @@ -118,14 +125,8 @@ class ProjectTeam max_member_access(user.id) == Gitlab::Access::MASTER end - def member?(user, min_member_access = nil) - member = !!find_member(user.id) - - if min_member_access - member && max_member_access(user.id) >= min_member_access - else - member - end + def member?(user, min_member_access = Gitlab::Access::GUEST) + max_member_access(user.id) >= min_member_access end def human_max_access(user_id) @@ -162,7 +163,7 @@ class ProjectTeam # Each group produces a list of maximum access level per user. We take the # max of the values produced by each group. - if project.invited_groups.any? && project.allowed_to_share_with_group? + if project_shared_with_group? project.project_group_links.each do |group_link| invited_access = max_invited_level_for_users(group_link, user_ids) merge_max!(access, invited_access) @@ -199,43 +200,17 @@ class ProjectTeam def fetch_members(level = nil) project_members = project.members group_members = group ? group.members : [] - invited_members = [] - - if project.invited_groups.any? && project.allowed_to_share_with_group? - project.project_group_links.includes(group: [:group_members]).each do |group_link| - invited_group = group_link.group - im = invited_group.members - - if level - int_level = GroupMember.access_level_roles[level.to_s.singularize.titleize] - - # Skip group members if we ask for masters - # but max group access is developers - next if int_level > group_link.group_access - - # If we ask for developers and max - # group access is developers we need to provide - # both group master, developers as devs - if int_level == group_link.group_access - im.where("access_level >= ?)", group_link.group_access) - else - im.send(level) - end - end - - invited_members << im - end - - invited_members = invited_members.flatten.compact - end if level - project_members = project_members.send(level) - group_members = group_members.send(level) if group + project_members = project_members.public_send(level) + group_members = group_members.public_send(level) if group end user_ids = project_members.pluck(:user_id) + + invited_members = fetch_invited_members(level) user_ids.push(*invited_members.map(&:user_id)) if invited_members.any? + user_ids.push(*group_members.pluck(:user_id)) if group User.where(id: user_ids) @@ -248,4 +223,38 @@ class ProjectTeam def merge_max!(first_hash, second_hash) first_hash.merge!(second_hash) { |_key, old, new| old > new ? old : new } end + + def project_shared_with_group? + project.invited_groups.any? && project.allowed_to_share_with_group? + end + + def fetch_invited_members(level = nil) + invited_members = [] + + return invited_members unless project_shared_with_group? + + project.project_group_links.includes(group: [:group_members]).each do |link| + invited_group_members = link.group.members + + if level + numeric_level = GroupMember.access_level_roles[level.to_s.singularize.titleize] + + # If we're asked for a level that's higher than the group's access, + # there's nothing left to do + next if numeric_level > link.group_access + + # Make sure we include everyone _above_ the requested level as well + invited_group_members = + if numeric_level == link.group_access + invited_group_members.where("access_level >= ?", link.group_access) + else + invited_group_members.public_send(level) + end + end + + invited_members << invited_group_members + end + + invited_members.flatten.compact + end end diff --git a/app/models/release.rb b/app/models/release.rb index e196b84eb18..c936899799e 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -1,4 +1,8 @@ class Release < ActiveRecord::Base + include CacheMarkdownField + + cache_markdown_field :description + belongs_to :project validates :description, :project, :tag, presence: true diff --git a/app/models/repository.rb b/app/models/repository.rb index e56bac509a4..fe991904601 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -11,6 +11,20 @@ class Repository attr_accessor :path_with_namespace, :project + def self.storages + Gitlab.config.repositories.storages + end + + def self.remove_storage_from_path(repo_path) + storages.find do |_, storage_path| + if repo_path.start_with?(storage_path) + return repo_path.sub(storage_path, '') + end + end + + repo_path + end + def initialize(path_with_namespace, project) @path_with_namespace = path_with_namespace @project = project @@ -70,15 +84,17 @@ class Repository def commit(ref = 'HEAD') return nil unless exists? + commit = if ref.is_a?(Gitlab::Git::Commit) ref else Gitlab::Git::Commit.find(raw_repository, ref) end + commit = ::Commit.new(commit, @project) if commit commit - rescue Rugged::OdbError + rescue Rugged::OdbError, Rugged::TreeError nil end @@ -109,19 +125,37 @@ class Repository end def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0) + unless exists? && has_visible_content? && query.present? + return [] + end + ref ||= root_ref - # Limited to 1000 commits for now, could be parameterized? - args = %W(#{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} --max-count #{limit} --grep=#{query}) + args = %W( + #{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} + --max-count #{limit} --grep=#{query} --regexp-ignore-case + ) args = args.concat(%W(-- #{path})) if path.present? - git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:chomp) - commits = git_log_results.map { |c| commit(c) } - commits + git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines + git_log_results.map { |c| commit(c.chomp) }.compact end - def find_branch(name) - raw_repository.branches.find { |branch| branch.name == name } + def find_branch(name, fresh_repo: true) + # Since the Repository object may have in-memory index changes, invalidating the memoized Repository object may + # cause unintended side effects. Because finding a branch is a read-only operation, we can safely instantiate + # a new repo here to ensure a consistent state to avoid a libgit2 bug where concurrent access (e.g. via git gc) + # may cause the branch to "disappear" erroneously or have the wrong SHA. + # + # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392 + raw_repo = + if fresh_repo + Gitlab::Git::Repository.new(path_to_repo) + else + raw_repository + end + + raw_repo.find_branch(name) end def find_tag(name) @@ -136,7 +170,7 @@ class Repository return false unless target GitHooksService.new.execute(user, path_to_repo, oldrev, target, ref) do - rugged.branches.create(branch_name, target) + update_ref!(ref, target, oldrev) end after_create_branch @@ -163,12 +197,12 @@ class Repository before_remove_branch branch = find_branch(branch_name) - oldrev = branch.try(:target).try(:id) + oldrev = branch.try(:dereferenced_target).try(:id) newrev = Gitlab::Git::BLANK_SHA ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name GitHooksService.new.execute(user, path_to_repo, oldrev, newrev, ref) do - rugged.branches.delete(branch_name) + update_ref!(ref, newrev, oldrev) end after_remove_branch @@ -200,6 +234,23 @@ class Repository def ref_exists?(ref) rugged.references.exist?(ref) + rescue Rugged::ReferenceError + false + end + + def update_ref!(name, newrev, oldrev) + # We use 'git update-ref' because libgit2/rugged currently does not + # offer 'compare and swap' ref updates. Without compare-and-swap we can + # (and have!) accidentally reset the ref to an earlier state, clobbering + # commits. See also https://github.com/libgit2/libgit2/issues/1534. + command = %w[git update-ref --stdin -z] + _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin| + stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00") + end + + return if status.zero? + + raise CommitError.new("Could not update branch #{name.sub('refs/heads/', '')}. Please refresh and try again.") end # Makes sure a commit is kept around when Git garbage collection runs. @@ -223,11 +274,7 @@ class Repository end def kept_around?(sha) - begin - ref_exists?(keep_around_ref_name(sha)) - rescue Rugged::ReferenceError - false - end + ref_exists?(keep_around_ref_name(sha)) end def tag_names @@ -264,10 +311,10 @@ class Repository # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes number_commits_behind = raw_repository. - count_commits_between(branch.target.sha, root_ref_hash) + count_commits_between(branch.dereferenced_target.sha, root_ref_hash) number_commits_ahead = raw_repository. - count_commits_between(root_ref_hash, branch.target.sha) + count_commits_between(root_ref_hash, branch.dereferenced_target.sha) { behind: number_commits_behind, ahead: number_commits_ahead } end @@ -277,7 +324,7 @@ class Repository def cache_keys %i(size commit_count readme version contribution_guide changelog - license_blob license_key gitignore) + license_blob license_key gitignore koding_yml) end # Keys for data on branch/tag operations. @@ -386,11 +433,24 @@ class Repository @exists = nil end + # expire cache that doesn't depend on repository data (when expiring) + def expire_content_cache + expire_tags_cache + expire_tag_count_cache + expire_branches_cache + expire_branch_count_cache + expire_root_ref_cache + expire_emptiness_caches + expire_exists_cache + end + # Runs code after a repository has been created. def after_create expire_exists_cache expire_root_ref_cache expire_emptiness_caches + + repository_event(:create_repository) end # Runs code just before a repository is deleted. @@ -399,14 +459,9 @@ class Repository expire_cache if exists? - # expire cache that don't depend on repository data (when expiring) - expire_tags_cache - expire_tag_count_cache - expire_branches_cache - expire_branch_count_cache - expire_root_ref_cache - expire_emptiness_caches - expire_exists_cache + expire_content_cache + + repository_event(:remove_repository) end # Runs code just before the HEAD of a repository is changed. @@ -414,6 +469,8 @@ class Repository # Cached divergent commit counts are based on repository head expire_branch_cache expire_root_ref_cache + + repository_event(:change_default_branch) end # Runs code before pushing (= creating or removing) a tag. @@ -421,28 +478,33 @@ class Repository expire_cache expire_tags_cache expire_tag_count_cache + + repository_event(:push_tag) end # Runs code before removing a tag. def before_remove_tag expire_tags_cache expire_tag_count_cache + + repository_event(:remove_tag) end def before_import - expire_emptiness_caches - expire_exists_cache + expire_content_cache end # Runs code after a repository has been forked/imported. def after_import - expire_emptiness_caches - expire_exists_cache + expire_content_cache + build_cache end # Runs code after a new commit has been pushed. def after_push_commit(branch_name, revision) expire_cache(branch_name, revision) + + repository_event(:push_commit, branch: branch_name) end # Runs code after a new branch has been created. @@ -450,11 +512,15 @@ class Repository expire_branches_cache expire_has_visible_content_cache expire_branch_count_cache + + repository_event(:push_branch) end # Runs code before removing an existing branch. def before_remove_branch expire_branches_cache + + repository_event(:remove_branch) end # Runs code after an existing branch has been removed. @@ -537,6 +603,14 @@ class Repository end end + def koding_yml + return nil unless head_exists? + + cache.fetch(:koding_yml) do + file_on_head(/\A\.koding\.yml\z/) + end + end + def gitlab_ci_yml return nil unless head_exists? @@ -622,11 +696,11 @@ class Repository branches.sort_by(&:name) when 'updated_desc' branches.sort do |a, b| - commit(b.target).committed_date <=> commit(a.target).committed_date + commit(b.dereferenced_target).committed_date <=> commit(a.dereferenced_target).committed_date end when 'updated_asc' branches.sort do |a, b| - commit(a.target).committed_date <=> commit(b.target).committed_date + commit(a.dereferenced_target).committed_date <=> commit(b.dereferenced_target).committed_date end else branches @@ -665,6 +739,14 @@ class Repository end end + def ref_name_for_sha(ref_path, sha) + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, path_to_repo).first.split.last + end + def refs_contains_sha(ref_type, sha) args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha}) names = Gitlab::Popen.popen(args, path_to_repo).first @@ -704,64 +786,61 @@ class Repository @root_ref ||= cache.fetch(:root_ref) { raw_repository.root_ref } end - def commit_dir(user, path, message, branch) - commit_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + def commit_dir(user, path, message, branch, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + options = { + commit: { + branch: ref, + message: message, + update_ref: false + } } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) + raw_repository.mkdir(path, options) end end - def commit_file(user, path, content, message, branch, update) - commit_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + def commit_file(user, path, content, message, branch, update, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + content: content, + path: path, + update: update + } } - options[:file] = { - content: content, - path: path, - update: update - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) Gitlab::Git::Blob.commit(raw_repository, options) end end - def update_file(user, path, content, branch:, previous_path:, message:) - commit_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false + def update_file(user, path, content, branch:, previous_path:, message:, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + content: content, + path: path, + update: true + } } - options[:file] = { - content: content, - path: path, - update: true - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) - if previous_path + if previous_path && previous_path != path options[:file][:previous_path] = previous_path Gitlab::Git::Blob.rename(raw_repository, options) else @@ -770,34 +849,85 @@ class Repository end end - def remove_file(user, path, message, branch) - commit_with_hooks(user, branch) do |ref| - committer = user_to_committer(user) - options = {} - options[:committer] = committer - options[:author] = committer - options[:commit] = { - message: message, - branch: ref, - update_ref: false, + def remove_file(user, path, message, branch, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + options = { + commit: { + branch: ref, + message: message, + update_ref: false + }, + file: { + path: path + } } - options[:file] = { - path: path - } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) Gitlab::Git::Blob.remove(raw_repository, options) end end - def user_to_committer(user) + def multi_action(user:, branch:, message:, actions:, author_email: nil, author_name: nil) + update_branch_with_hooks(user, branch) do |ref| + index = rugged.index + parents = [] + branch = find_branch(ref) + + if branch + last_commit = branch.dereferenced_target + index.read_tree(last_commit.raw_commit.tree) + parents = [last_commit.sha] + end + + actions.each do |action| + case action[:action] + when :create, :update, :move + mode = + case action[:action] + when :update + index.get(action[:file_path])[:mode] + when :move + index.get(action[:previous_path])[:mode] + end + mode ||= 0o100644 + + index.remove(action[:previous_path]) if action[:action] == :move + + content = action[:encoding] == 'base64' ? Base64.decode64(action[:content]) : action[:content] + oid = rugged.write(content, :blob) + + index.add(path: action[:file_path], oid: oid, mode: mode) + when :delete + index.remove(action[:file_path]) + end + end + + options = { + tree: index.write_tree(rugged), + message: message, + parents: parents + } + options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) + + Rugged::Commit.create(rugged, options) + end + end + + def get_committer_and_author(user, email: nil, name: nil) + committer = user_to_committer(user) + author = Gitlab::Git::committer_hash(email: email, name: name) || committer + { - email: user.email, - name: user.name, - time: Time.now + author: author, + committer: committer } end + def user_to_committer(user) + Gitlab::Git::committer_hash(email: user.email, name: user.name) + end + def can_be_merged?(source_sha, target_branch) our_commit = rugged.branches[target_branch].target their_commit = rugged.lookup(source_sha) @@ -819,7 +949,7 @@ class Repository merge_index = rugged.merge_commits(our_commit, their_commit) return false if merge_index.conflicts? - commit_with_hooks(user, merge_request.target_branch) do + update_branch_with_hooks(user, merge_request.target_branch) do actual_options = options.merge( parents: [our_commit, their_commit], tree: merge_index.write_tree(rugged), @@ -832,12 +962,12 @@ class Repository end def revert(user, commit, base_branch, revert_tree_id = nil) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha revert_tree_id ||= check_revert_content(commit, base_branch) return false unless revert_tree_id - commit_with_hooks(user, base_branch) do + update_branch_with_hooks(user, base_branch) do committer = user_to_committer(user) source_sha = Rugged::Commit.create(rugged, message: commit.revert_message, @@ -849,12 +979,12 @@ class Repository end def cherry_pick(user, commit, base_branch, cherry_pick_tree_id = nil) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha cherry_pick_tree_id ||= check_cherry_pick_content(commit, base_branch) return false unless cherry_pick_tree_id - commit_with_hooks(user, base_branch) do + update_branch_with_hooks(user, base_branch) do committer = user_to_committer(user) source_sha = Rugged::Commit.create(rugged, message: commit.message, @@ -869,8 +999,16 @@ class Repository end end + def resolve_conflicts(user, branch, params) + update_branch_with_hooks(user, branch) do + committer = user_to_committer(user) + + Rugged::Commit.create(rugged, params.merge(author: committer, committer: committer)) + end + end + def check_revert_content(commit, base_branch) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha args = [commit.id, source_sha] args << { mainline: 1 } if commit.merge_commit? @@ -884,7 +1022,7 @@ class Repository end def check_cherry_pick_content(commit, base_branch) - source_sha = find_branch(base_branch).target.sha + source_sha = find_branch(base_branch).dereferenced_target.sha args = [commit.id, source_sha] args << 1 if commit.merge_commit? @@ -906,7 +1044,8 @@ class Repository root_ref_commit = commit(root_ref) if branch_commit - is_ancestor?(branch_commit.id, root_ref_commit.id) + same_head = branch_commit.id == root_ref_commit.id + !same_head && is_ancestor?(branch_commit.id, root_ref_commit.id) else nil end @@ -925,59 +1064,31 @@ class Repository end def search_files(query, ref) + unless exists? && has_visible_content? && query.present? + return [] + end + offset = 2 args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/) end - def parse_search_result(result) - ref = nil - filename = nil - basename = nil - startline = 0 - - result.each_line.each_with_index do |line, index| - if line =~ /^.*:.*:\d+:/ - ref, filename, startline = line.split(':') - startline = startline.to_i - index - extname = Regexp.escape(File.extname(filename)) - basename = filename.sub(/#{extname}$/, '') - break - end - end - - data = "" - - result.each_line do |line| - data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') - end - - OpenStruct.new( - filename: filename, - basename: basename, - ref: ref, - startline: startline, - data: data - ) - end - def fetch_ref(source_path, source_ref, target_ref) args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) Gitlab::Popen.popen(args, path_to_repo) end - def commit_with_hooks(current_user, branch) + def create_ref(ref, ref_path) + fetch_ref(path_to_repo, ref, ref_path) + end + + def update_branch_with_hooks(current_user, branch) update_autocrlf_option - oldrev = Gitlab::Git::BLANK_SHA ref = Gitlab::Git::BRANCH_REF_PREFIX + branch target_branch = find_branch(branch) was_empty = empty? - if !was_empty && target_branch - oldrev = target_branch.target.id - end - # Make commit newrev = yield(ref) @@ -985,24 +1096,19 @@ class Repository raise CommitError.new('Failed to create commit') end + if rugged.lookup(newrev).parent_ids.empty? || target_branch.nil? + oldrev = Gitlab::Git::BLANK_SHA + else + oldrev = rugged.merge_base(newrev, target_branch.dereferenced_target.sha) + end + GitHooksService.new.execute(current_user, path_to_repo, oldrev, newrev, ref) do - if was_empty || !target_branch - # Create branch - rugged.references.create(ref, newrev) + update_ref!(ref, newrev, oldrev) + if was_empty || !target_branch # If repo was empty expire cache after_create if was_empty after_create_branch - else - # Update head - current_head = find_branch(branch).target.id - - # Make sure target branch was not changed during pre-receive hook - if current_head == oldrev - rugged.references.update(ref, newrev) - else - raise CommitError.new('Commit was rejected because branch received new push') - end end end @@ -1033,7 +1139,7 @@ class Repository @avatar ||= cache.fetch(:avatar) do AVATAR_FILES.find do |file| - blob_at_branch('master', file) + blob_at_branch(root_ref, file) end end end @@ -1053,10 +1159,14 @@ class Repository end def tags_sorted_by_committed_date - tags.sort_by { |tag| tag.target.committed_date } + tags.sort_by { |tag| tag.dereferenced_target.committed_date } end def keep_around_ref_name(sha) "refs/keep-around/#{sha}" end + + def repository_event(event, tags = {}) + Gitlab::Metrics.add_event(event, { path: path_with_namespace }.merge(tags)) + end end diff --git a/app/models/service.rb b/app/models/service.rb index 09b4717a523..625fbc48302 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -7,10 +7,12 @@ class Service < ActiveRecord::Base default_value_for :active, false default_value_for :push_events, true default_value_for :issues_events, true + default_value_for :confidential_issues_events, true default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true default_value_for :build_events, true + default_value_for :pipeline_events, true default_value_for :wiki_page_events, true after_initialize :initialize_properties @@ -33,6 +35,7 @@ class Service < ActiveRecord::Base scope :push_hooks, -> { where(push_events: true, active: true) } scope :tag_push_hooks, -> { where(tag_push_events: true, active: true) } scope :issue_hooks, -> { where(issues_events: true, active: true) } + scope :confidential_issue_hooks, -> { where(confidential_issues_events: true, active: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true, active: true) } scope :note_hooks, -> { where(note_events: true, active: true) } scope :build_hooks, -> { where(build_events: true, active: true) } @@ -100,7 +103,7 @@ class Service < ActiveRecord::Base end def supported_events - %w(push tag_push issue merge_request wiki_page) + %w(push tag_push issue confidential_issue merge_request wiki_page) end def execute(data) @@ -133,6 +136,7 @@ class Service < ActiveRecord::Base end def #{arg}=(value) + self.properties ||= {} updated_properties['#{arg}'] = #{arg} unless #{arg}_changed? self.properties['#{arg}'] = value end @@ -192,12 +196,13 @@ class Service < ActiveRecord::Base end def self.available_services_names - %w( + %w[ asana assembla bamboo buildkite builds_email + pipelines_email bugzilla campfire custom_issue_tracker @@ -214,7 +219,7 @@ class Service < ActiveRecord::Base redmine slack teamcity - ) + ] end def self.create_from_template(project_id, template) diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 5ec933601ac..2373b445009 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -1,9 +1,20 @@ class Snippet < ActiveRecord::Base include Gitlab::VisibilityLevel include Linguist::BlobHelper + include CacheMarkdownField include Participable include Referable include Sortable + include Awardable + + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :content + + # If file_name changes, it invalidates content + alias_method :default_content_html_invalidator, :content_html_invalidated? + def content_html_invalidated? + default_content_html_invalidator || file_name_changed? + end default_value_for :visibility_level, Snippet::PRIVATE diff --git a/app/models/todo.rb b/app/models/todo.rb index 8d7a5965aa1..f5ade1cc293 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -1,4 +1,6 @@ class Todo < ActiveRecord::Base + include Sortable + ASSIGNED = 1 MENTIONED = 2 BUILD_FAILED = 3 @@ -41,6 +43,29 @@ class Todo < ActiveRecord::Base after_save :keep_around_commit + class << self + def sort(method) + method == "priority" ? order_by_labels_priority : order_by(method) + end + + # Order by priority depending on which issue/merge request the Todo belongs to + # Todos with highest priority first then oldest todos + # Need to order by created_at last because of differences on Mysql and Postgres when joining by type "Merge_request/Issue" + def order_by_labels_priority + params = { + target_type_column: "todos.target_type", + target_column: "todos.target_id", + project_column: "todos.project_id" + } + + highest_priority = highest_label_priority(params).to_sql + + select("#{table_name}.*, (#{highest_priority}) AS highest_priority"). + order(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')). + order('todos.created_at') + end + end + def build_failed? action == BUILD_FAILED end diff --git a/app/models/trending_project.rb b/app/models/trending_project.rb new file mode 100644 index 00000000000..27e3732da17 --- /dev/null +++ b/app/models/trending_project.rb @@ -0,0 +1,35 @@ +class TrendingProject < ActiveRecord::Base + belongs_to :project + + # The number of months to include in the trending calculation. + MONTHS_TO_INCLUDE = 1 + + # The maximum number of projects to include in the trending set. + PROJECTS_LIMIT = 100 + + # Populates the trending projects table with the current list of trending + # projects. + def self.refresh! + # The calculation **must** run in a transaction. If the removal of data and + # insertion of new data were to run separately a user might end up with an + # empty list of trending projects for a short period of time. + transaction do + delete_all + + timestamp = connection.quote(MONTHS_TO_INCLUDE.months.ago) + + connection.execute <<-EOF.strip_heredoc + INSERT INTO #{table_name} (project_id) + SELECT project_id + FROM notes + INNER JOIN projects ON projects.id = notes.project_id + WHERE notes.created_at >= #{timestamp} + AND notes.system IS FALSE + AND projects.visibility_level = #{Gitlab::VisibilityLevel::PUBLIC} + GROUP BY project_id + ORDER BY count(*) DESC + LIMIT #{PROJECTS_LIMIT}; + EOF + end + end +end diff --git a/app/models/u2f_registration.rb b/app/models/u2f_registration.rb index 00b19686d48..808acec098f 100644 --- a/app/models/u2f_registration.rb +++ b/app/models/u2f_registration.rb @@ -3,18 +3,19 @@ class U2fRegistration < ActiveRecord::Base belongs_to :user - def self.register(user, app_id, json_response, challenges) + def self.register(user, app_id, params, challenges) u2f = U2F::U2F.new(app_id) registration = self.new begin - response = U2F::RegisterResponse.load_from_json(json_response) + response = U2F::RegisterResponse.load_from_json(params[:device_response]) registration_data = u2f.register!(challenges, response) registration.update(certificate: registration_data.certificate, key_handle: registration_data.key_handle, public_key: registration_data.public_key, counter: registration_data.counter, - user: user) + user: user, + name: params[:name]) rescue JSON::ParserError, NoMethodError, ArgumentError registration.errors.add(:base, 'Your U2F device did not send a valid JSON response.') rescue U2F::Error => e diff --git a/app/models/user.rb b/app/models/user.rb index 48e83ab7e56..3813df6684e 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -13,6 +13,7 @@ class User < ActiveRecord::Base DEFAULT_NOTIFICATION_LEVEL = :participating add_authentication_token_field :authentication_token + add_authentication_token_field :incoming_email_token default_value_for :admin, false default_value_for(:external) { current_application_settings.user_default_external } @@ -47,7 +48,7 @@ class User < ActiveRecord::Base # # Namespace for personal projects - has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id, class_name: "Namespace" + has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id # Profile has_many :keys, dependent: :destroy @@ -66,17 +67,17 @@ class User < ActiveRecord::Base # Projects has_many :groups_projects, through: :groups, source: :projects has_many :personal_projects, through: :namespace, source: :projects - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, class_name: 'ProjectMember' + has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy has_many :projects, through: :project_members has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy has_many :starred_projects, through: :users_star_projects, source: :project - has_many :snippets, dependent: :destroy, foreign_key: :author_id, class_name: "Snippet" + has_many :snippets, dependent: :destroy, foreign_key: :author_id has_many :issues, dependent: :destroy, foreign_key: :author_id has_many :notes, dependent: :destroy, foreign_key: :author_id has_many :merge_requests, dependent: :destroy, foreign_key: :author_id - has_many :events, dependent: :destroy, foreign_key: :author_id, class_name: "Event" + has_many :events, dependent: :destroy, foreign_key: :author_id has_many :subscriptions, dependent: :destroy has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" has_many :assigned_issues, dependent: :destroy, foreign_key: :assignee_id, class_name: "Issue" @@ -93,8 +94,10 @@ class User < ActiveRecord::Base # # Validations # + # Note: devise :validatable above adds validations for :email and :password validates :name, presence: true - validates :notification_email, presence: true, email: true + validates :notification_email, presence: true + validates :notification_email, email: true, if: ->(user) { user.notification_email != user.email } validates :public_email, presence: true, uniqueness: true, email: true, allow_blank: true validates :bio, length: { maximum: 255 }, allow_blank: true validates :projects_limit, presence: true, numericality: { greater_than_or_equal_to: 0 } @@ -117,7 +120,7 @@ class User < ActiveRecord::Base before_validation :set_public_email, if: ->(user) { user.public_email_changed? } after_update :update_emails_with_primary_email, if: ->(user) { user.email_changed? } - before_save :ensure_authentication_token + before_save :ensure_authentication_token, :ensure_incoming_email_token before_save :ensure_external_user_rights after_save :ensure_namespace_correct after_initialize :set_projects_limit @@ -171,6 +174,7 @@ class User < ActiveRecord::Base scope :active, -> { with_state(:active) } scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all } scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members)') } + scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) } def self.with_two_factor joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id"). @@ -256,6 +260,24 @@ class User < ActiveRecord::Base ) end + # searches user by given pattern + # it compares name, email, username fields and user's secondary emails with given pattern + # This method uses ILIKE on PostgreSQL and LIKE on MySQL. + + def search_with_secondary_emails(query) + table = arel_table + email_table = Email.arel_table + pattern = "%#{query}%" + matched_by_emails_user_ids = email_table.project(email_table[:user_id]).where(email_table[:email].matches(pattern)) + + where( + table[:name].matches(pattern). + or(table[:email].matches(pattern)). + or(table[:username].matches(pattern)). + or(table[:id].in(matched_by_emails_user_ids)) + ) + end + def by_login(login) return nil unless login @@ -279,6 +301,11 @@ class User < ActiveRecord::Base find_by('users.username = ? OR users.id = ?', name_or_id.to_s, name_or_id.to_i) end + # Returns a user for the given SSH key. + def find_by_ssh_key_id(key_id) + find_by(id: Key.unscoped.select(:user_id).where(id: key_id)) + end + def build_user(attrs = {}) User.new(attrs) end @@ -304,7 +331,7 @@ class User < ActiveRecord::Base username end - def to_reference(_from_project = nil) + def to_reference(_from_project = nil, _target_project = nil) "#{self.class.reference_prefix}#{username}" end @@ -418,6 +445,16 @@ class User < ActiveRecord::Base Project.where("projects.id IN (#{projects_union(min_access_level).to_sql})") end + # Returns the projects this user has reporter (or greater) access to, limited + # to at most the given projects. + # + # This method is useful when you have a list of projects and want to + # efficiently check to which of these projects the user has at least reporter + # access. + def projects_with_reporter_access_limited_to(projects) + authorized_projects(Gitlab::Access::REPORTER).where(id: projects) + end + def viewable_starred_projects starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (#{projects_union.to_sql})", [Project::PUBLIC, Project::INTERNAL]) @@ -433,7 +470,7 @@ class User < ActiveRecord::Base # # This logic is duplicated from `Ability#project_abilities` into a SQL form. def projects_where_can_admin_issues - authorized_projects(Gitlab::Access::REPORTER).non_archived.where.not(issues_enabled: false) + authorized_projects(Gitlab::Access::REPORTER).non_archived.with_issues_enabled end def is_admin? @@ -460,16 +497,12 @@ class User < ActiveRecord::Base can?(:create_group, nil) end - def abilities - Ability.abilities - end - def can_select_namespace? several_namespaces? || admin end def can?(action, subject) - abilities.allowed?(self, action, subject) + Ability.allowed?(self, action, subject) end def first_name @@ -489,10 +522,10 @@ class User < ActiveRecord::Base (personal_projects.count.to_f / projects_limit) * 100 end - def recent_push(project_id = nil) + def recent_push(project_ids = nil) # Get push events not earlier than 2 hours ago events = recent_events.code_push.where("created_at > ?", Time.now - 2.hours) - events = events.where(project_id: project_id) if project_id + events = events.where(project_id: project_ids) if project_ids # Use the latest event that has not been pushed or merged recently events.recent.find do |event| @@ -588,6 +621,11 @@ class User < ActiveRecord::Base end def set_projects_limit + # `User.select(:id)` raises + # `ActiveModel::MissingAttributeError: missing attribute: projects_limit` + # without this safeguard! + return unless self.has_attribute?(:projects_limit) + connection_default_value_defined = new_record? && !projects_limit_changed? return unless self.projects_limit.nil? || connection_default_value_defined @@ -831,6 +869,22 @@ class User < ActiveRecord::Base todos_pending_count(force: true) end + # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth + # flow means we don't call that automatically (and can't conveniently do so). + # + # See: + # <https://github.com/plataformatec/devise/blob/v4.0.0/lib/devise/models/lockable.rb#L92> + # + def increment_failed_attempts! + self.failed_attempts ||= 0 + self.failed_attempts += 1 + if attempts_exceeded? + lock_access! unless access_locked? + else + save(validate: false) + end + end + private def projects_union(min_access_level = nil) @@ -885,7 +939,7 @@ class User < ActiveRecord::Base if domain_matches?(allowed_domains, self.email) valid = true else - error = "is not whitelisted. Email domains valid for registration are: #{allowed_domains.join(', ')}" + error = "domain is not authorized for sign-up" valid = false end end @@ -903,4 +957,13 @@ class User < ActiveRecord::Base signup_domain =~ regexp end end + + def generate_token(token_field) + if token_field == :incoming_email_token + # Needs to be all lowercase and alphanumeric because it's gonna be used in an email address. + SecureRandom.hex.to_i(16).to_s(36) + else + super + end + end end |