diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-05-23 02:10:29 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-05-23 02:10:29 +0800 |
| commit | 1a4130d3a6cfb4956f8bb1186cc499ea549d8e18 (patch) | |
| tree | 076adcb3e6f3800a1a7bbc6809839d5cb3b3f372 /app/models | |
| parent | 3c8a6fba67998eb17240b15db85f8d1c8aff338e (diff) | |
| parent | 18a6d9c5326bc2b90a1f0cc8664d638a39885924 (diff) | |
| download | gitlab-ce-27377-preload-pipeline-entity.tar.gz | |
Merge remote-tracking branch 'upstream/master' into 27377-preload-pipeline-entity27377-preload-pipeline-entity
* upstream/master: (2534 commits)
Update VERSION to 9.3.0-pre
Update CHANGELOG.md for 9.2.0
removes unnecessary redundacy in usage ping doc
Respect the typo as rubocop said
Add a test to ensure this works on MySQL
Change pipelines schedules help page path
change domain to hostname in usage ping doc
Fixes broken MySQL migration for retried
Show password field mask while editing service settings
Add notes for supported schedulers and cloud providers
Move environment monitoring to environments doc
Add docs for change of Cache/Artifact restore order"
Avoid resource intensive login checks if password is not provided
Change translation for 'coding' by 'desarrollo' for Spanish
Add to docs: issues multiple assignees
rename "Add emoji" and "Award emoji" to "Add reaction" where appropriate
Add project and group notification settings info
32570 Fix border-bottom for project activity tab
Add users endpoint to frontend API class
Rename users on mysql
...
Diffstat (limited to 'app/models')
154 files changed, 3515 insertions, 1432 deletions
diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index 2340453831e..0d7c2d20029 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -16,7 +16,7 @@ class AbuseReport < ActiveRecord::Base def remove_user(deleted_by:) user.block - DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true) + DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true, hard_delete: true) end def notify diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 671a0fe98cc..043f57241a3 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -28,6 +28,8 @@ class ApplicationSetting < ActiveRecord::Base attr_accessor :domain_whitelist_raw, :domain_blacklist_raw + validates :uuid, presence: true + validates :session_expire_delay, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -60,6 +62,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, if: :sentry_enabled + validates :clientside_sentry_dsn, + presence: true, + if: :clientside_sentry_enabled + validates :akismet_api_key, presence: true, if: :akismet_enabled @@ -131,6 +137,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :polling_interval_multiplier, + presence: true, + numericality: { greater_than_or_equal_to: 0 } + validates_each :restricted_visibility_levels do |record, attr, value| value&.each do |level| unless Gitlab::VisibilityLevel.options.has_value?(level) @@ -155,6 +165,7 @@ class ApplicationSetting < ActiveRecord::Base end end + before_validation :ensure_uuid! before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -233,7 +244,9 @@ class ApplicationSetting < ActiveRecord::Base signup_enabled: Settings.gitlab['signup_enabled'], terminal_max_session_time: 0, two_factor_grace_period: 48, - user_default_external: false + user_default_external: false, + polling_interval_multiplier: 1, + usage_ping_enabled: Settings.gitlab['usage_ping_enabled'] } end @@ -336,8 +349,22 @@ class ApplicationSetting < ActiveRecord::Base sidekiq_throttling_enabled end + def usage_ping_can_be_configured? + Settings.gitlab.usage_ping_enabled + end + + def usage_ping_enabled + usage_ping_can_be_configured? && super + end + private + def ensure_uuid! + return if uuid? + + self.uuid = SecureRandom.uuid + end + def check_repository_storages invalid = repository_storages - Gitlab.config.repositories.storages.keys errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 6937ad3bdd9..6ada6fae4eb 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -3,13 +3,14 @@ class AwardEmoji < ActiveRecord::Base UPVOTE_NAME = "thumbsup".freeze include Participable + include GhostUser belongs_to :awardable, polymorphic: true belongs_to :user validates :awardable, :user, presence: true validates :name, presence: true, inclusion: { in: Gitlab::Emoji.emojis_names } - validates :name, uniqueness: { scope: [:user, :awardable_type, :awardable_id] } + validates :name, uniqueness: { scope: [:user, :awardable_type, :awardable_id] }, unless: :ghost_user? participant :user diff --git a/app/models/blob.rb b/app/models/blob.rb index 95d2111a992..e75926241ba 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -3,8 +3,62 @@ class Blob < SimpleDelegator CACHE_TIME = 60 # Cache raw blobs referred to by a (mutable) ref for 1 minute CACHE_TIME_IMMUTABLE = 3600 # Cache blobs referred to by an immutable reference for 1 hour - # The maximum size of an SVG that can be displayed. - MAXIMUM_SVG_SIZE = 2.megabytes + MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte + + # Finding a viewer for a blob happens based only on extension and whether the + # blob is binary or text, which means 1 blob should only be matched by 1 viewer, + # and the order of these viewers doesn't really matter. + # + # However, when the blob is an LFS pointer, we cannot know for sure whether the + # file being pointed to is binary or text. In this case, we match only on + # extension, preferring binary viewers over text ones if both exist, since the + # large files referred to in "Large File Storage" are much more likely to be + # binary than text. + # + # `.stl` files, for example, exist in both binary and text forms, and are + # handled by different viewers (`BinarySTL` and `TextSTL`) depending on blob + # type. LFS pointers to `.stl` files are assumed to always be the binary kind, + # and use the `BinarySTL` viewer. + RICH_VIEWERS = [ + BlobViewer::Markup, + BlobViewer::Notebook, + BlobViewer::SVG, + + BlobViewer::Image, + BlobViewer::Sketch, + BlobViewer::Balsamiq, + + BlobViewer::Video, + + BlobViewer::PDF, + + BlobViewer::BinarySTL, + BlobViewer::TextSTL + ].sort_by { |v| v.binary? ? 0 : 1 }.freeze + + AUXILIARY_VIEWERS = [ + BlobViewer::GitlabCiYml, + BlobViewer::RouteMap, + + BlobViewer::Readme, + BlobViewer::License, + BlobViewer::Contributing, + BlobViewer::Changelog, + + BlobViewer::Cartfile, + BlobViewer::ComposerJson, + BlobViewer::Gemfile, + BlobViewer::Gemspec, + BlobViewer::GodepsJson, + BlobViewer::PackageJson, + BlobViewer::Podfile, + BlobViewer::Podspec, + BlobViewer::PodspecJson, + BlobViewer::RequirementsTxt, + BlobViewer::YarnLock + ].freeze + + attr_reader :project # Wrap a Gitlab::Git::Blob object, or return nil when given nil # @@ -16,10 +70,16 @@ class Blob < SimpleDelegator # # blob = Blob.decorate(nil) # puts "truthy" if blob # No output - def self.decorate(blob) + def self.decorate(blob, project = nil) return if blob.nil? - new(blob) + new(blob, project) + end + + def initialize(blob, project = nil) + @project = project + + super(blob) end # Returns the data of the blob. @@ -35,44 +95,128 @@ class Blob < SimpleDelegator end def no_highlighting? - size && size > 1.megabyte + raw_size && raw_size > MAXIMUM_TEXT_HIGHLIGHT_SIZE end - def only_display_raw? - size && truncated? + def empty? + raw_size == 0 end - def svg? - text? && language && language.name == 'SVG' + def too_large? + size && truncated? end - def ipython_notebook? - text? && language&.name == 'Jupyter Notebook' + def external_storage_error? + if external_storage == :lfs + !project&.lfs_enabled? + else + false + end end - def size_within_svg_limits? - size <= MAXIMUM_SVG_SIZE + def stored_externally? + return @stored_externally if defined?(@stored_externally) + + @stored_externally = external_storage && !external_storage_error? end - def video? - UploaderHelper::VIDEO_EXT.include?(extname.downcase.delete('.')) + # Returns the size of the file that this blob represents. If this blob is an + # LFS pointer, this is the size of the file stored in LFS. Otherwise, this is + # the size of the blob itself. + def raw_size + if stored_externally? + external_size + else + size + end end - def to_partial_path(project) - if lfs_pointer? - if project.lfs_enabled? - 'download' + # Returns whether the file that this blob represents is binary. If this blob is + # an LFS pointer, we assume the file stored in LFS is binary, unless a + # text-based rich blob viewer matched on the file's extension. Otherwise, this + # depends on the type of the blob itself. + def raw_binary? + if stored_externally? + if rich_viewer + rich_viewer.binary? + elsif Linguist::Language.find_by_filename(name).any? + false + elsif _mime_type + _mime_type.binary? else - 'text' + true end - elsif image? || svg? - 'image' - elsif ipython_notebook? - 'notebook' - elsif text? - 'text' else - 'download' + binary? end end + + def extension + @extension ||= extname.downcase.delete('.') + end + + def video? + UploaderHelper::VIDEO_EXT.include?(extension) + end + + def readable_text? + text? && !stored_externally? && !too_large? + end + + def simple_viewer + @simple_viewer ||= simple_viewer_class.new(self) + end + + def rich_viewer + return @rich_viewer if defined?(@rich_viewer) + + @rich_viewer = rich_viewer_class&.new(self) + end + + def auxiliary_viewer + return @auxiliary_viewer if defined?(@auxiliary_viewer) + + @auxiliary_viewer = auxiliary_viewer_class&.new(self) + end + + def rendered_as_text?(ignore_errors: true) + simple_viewer.text? && (ignore_errors || simple_viewer.render_error.nil?) + end + + def show_viewer_switcher? + rendered_as_text? && rich_viewer + end + + def override_max_size! + simple_viewer&.override_max_size = true + rich_viewer&.override_max_size = true + end + + private + + def simple_viewer_class + if empty? + BlobViewer::Empty + elsif raw_binary? + BlobViewer::Download + else # text + BlobViewer::Text + end + end + + def rich_viewer_class + viewer_class_from(RICH_VIEWERS) + end + + def auxiliary_viewer_class + viewer_class_from(AUXILIARY_VIEWERS) + end + + def viewer_class_from(classes) + return if empty? || external_storage_error? + + verify_binary = !stored_externally? + + classes.find { |viewer_class| viewer_class.can_render?(self, verify_binary: verify_binary) } + end end diff --git a/app/models/blob_viewer/auxiliary.rb b/app/models/blob_viewer/auxiliary.rb new file mode 100644 index 00000000000..07a207730cf --- /dev/null +++ b/app/models/blob_viewer/auxiliary.rb @@ -0,0 +1,18 @@ +module BlobViewer + module Auxiliary + extend ActiveSupport::Concern + + include Gitlab::Allowable + + included do + self.loading_partial_name = 'loading_auxiliary' + self.type = :auxiliary + self.overridable_max_size = 100.kilobytes + self.max_size = 100.kilobytes + end + + def visible_to?(current_user) + true + end + end +end diff --git a/app/models/blob_viewer/balsamiq.rb b/app/models/blob_viewer/balsamiq.rb new file mode 100644 index 00000000000..f982521db99 --- /dev/null +++ b/app/models/blob_viewer/balsamiq.rb @@ -0,0 +1,12 @@ +module BlobViewer + class Balsamiq < Base + include Rich + include ClientSide + + self.partial_name = 'balsamiq' + self.extensions = %w(bmpr) + self.binary = true + self.switcher_icon = 'file-image-o' + self.switcher_title = 'preview' + end +end diff --git a/app/models/blob_viewer/base.rb b/app/models/blob_viewer/base.rb new file mode 100644 index 00000000000..26a3778c2a3 --- /dev/null +++ b/app/models/blob_viewer/base.rb @@ -0,0 +1,105 @@ +module BlobViewer + class Base + PARTIAL_PATH_PREFIX = 'projects/blob/viewers'.freeze + + class_attribute :partial_name, :loading_partial_name, :type, :extensions, :file_types, :load_async, :binary, :switcher_icon, :switcher_title, :overridable_max_size, :max_size + + self.loading_partial_name = 'loading' + + delegate :partial_path, :loading_partial_path, :rich?, :simple?, :text?, :binary?, to: :class + + attr_reader :blob + attr_accessor :override_max_size + + delegate :project, to: :blob + + def initialize(blob) + @blob = blob + end + + def self.partial_path + File.join(PARTIAL_PATH_PREFIX, partial_name) + end + + def self.loading_partial_path + File.join(PARTIAL_PATH_PREFIX, loading_partial_name) + end + + def self.rich? + type == :rich + end + + def self.simple? + type == :simple + end + + def self.auxiliary? + type == :auxiliary + end + + def self.load_async? + load_async + end + + def self.binary? + binary + end + + def self.text? + !binary? + end + + def self.can_render?(blob, verify_binary: true) + return false if verify_binary && binary? != blob.binary? + return true if extensions&.include?(blob.extension) + return true if file_types&.include?(Gitlab::FileDetector.type_of(blob.path)) + + false + end + + def load_async? + self.class.load_async? && render_error.nil? + end + + def exceeds_overridable_max_size? + overridable_max_size && blob.raw_size > overridable_max_size + end + + def exceeds_max_size? + max_size && blob.raw_size > max_size + end + + def can_override_max_size? + exceeds_overridable_max_size? && !exceeds_max_size? + end + + def too_large? + if override_max_size + exceeds_max_size? + else + exceeds_overridable_max_size? + end + end + + # This method is used on the server side to check whether we can attempt to + # render the blob at all. Human-readable error messages are found in the + # `BlobHelper#blob_render_error_reason` helper. + # + # This method does not and should not load the entire blob contents into + # memory, and should not be overridden to do so in order to validate the + # format of the blob. + # + # Prefer to implement a client-side viewer, where the JS component loads the + # binary from `blob_raw_url` and does its own format validation and error + # rendering, especially for potentially large binary formats. + def render_error + if too_large? + :too_large + end + end + + def prepare! + # To be overridden by subclasses + end + end +end diff --git a/app/models/blob_viewer/binary_stl.rb b/app/models/blob_viewer/binary_stl.rb new file mode 100644 index 00000000000..80393471ef2 --- /dev/null +++ b/app/models/blob_viewer/binary_stl.rb @@ -0,0 +1,10 @@ +module BlobViewer + class BinarySTL < Base + include Rich + include ClientSide + + self.partial_name = 'stl' + self.extensions = %w(stl) + self.binary = true + end +end diff --git a/app/models/blob_viewer/cartfile.rb b/app/models/blob_viewer/cartfile.rb new file mode 100644 index 00000000000..d8471bc33c0 --- /dev/null +++ b/app/models/blob_viewer/cartfile.rb @@ -0,0 +1,15 @@ +module BlobViewer + class Cartfile < DependencyManager + include Static + + self.file_types = %i(cartfile) + + def manager_name + 'Carthage' + end + + def manager_url + 'https://github.com/Carthage/Carthage' + end + end +end diff --git a/app/models/blob_viewer/changelog.rb b/app/models/blob_viewer/changelog.rb new file mode 100644 index 00000000000..0464ae27f71 --- /dev/null +++ b/app/models/blob_viewer/changelog.rb @@ -0,0 +1,16 @@ +module BlobViewer + class Changelog < Base + include Auxiliary + include Static + + self.partial_name = 'changelog' + self.file_types = %i(changelog) + self.binary = false + + def render_error + return if project.repository.tag_count > 0 + + :no_tags + end + end +end diff --git a/app/models/blob_viewer/client_side.rb b/app/models/blob_viewer/client_side.rb new file mode 100644 index 00000000000..cc68236f92b --- /dev/null +++ b/app/models/blob_viewer/client_side.rb @@ -0,0 +1,11 @@ +module BlobViewer + module ClientSide + extend ActiveSupport::Concern + + included do + self.load_async = false + self.overridable_max_size = 10.megabytes + self.max_size = 50.megabytes + end + end +end diff --git a/app/models/blob_viewer/composer_json.rb b/app/models/blob_viewer/composer_json.rb new file mode 100644 index 00000000000..ef8b4aef8e8 --- /dev/null +++ b/app/models/blob_viewer/composer_json.rb @@ -0,0 +1,23 @@ +module BlobViewer + class ComposerJson < DependencyManager + include ServerSide + + self.file_types = %i(composer_json) + + def manager_name + 'Composer' + end + + def manager_url + 'https://getcomposer.com/' + end + + def package_name + @package_name ||= package_name_from_json('name') + end + + def package_url + "https://packagist.org/packages/#{package_name}" + end + end +end diff --git a/app/models/blob_viewer/contributing.rb b/app/models/blob_viewer/contributing.rb new file mode 100644 index 00000000000..fbd1dd48697 --- /dev/null +++ b/app/models/blob_viewer/contributing.rb @@ -0,0 +1,10 @@ +module BlobViewer + class Contributing < Base + include Auxiliary + include Static + + self.partial_name = 'contributing' + self.file_types = %i(contributing) + self.binary = false + end +end diff --git a/app/models/blob_viewer/dependency_manager.rb b/app/models/blob_viewer/dependency_manager.rb new file mode 100644 index 00000000000..a8d9be945dc --- /dev/null +++ b/app/models/blob_viewer/dependency_manager.rb @@ -0,0 +1,43 @@ +module BlobViewer + class DependencyManager < Base + include Auxiliary + + self.partial_name = 'dependency_manager' + self.binary = false + + def manager_name + raise NotImplementedError + end + + def manager_url + raise NotImplementedError + end + + def package_type + 'package' + end + + def package_name + nil + end + + def package_url + nil + end + + private + + def package_name_from_json(key) + prepare! + + JSON.parse(blob.data)[key] rescue nil + end + + def package_name_from_method_call(name) + prepare! + + match = blob.data.match(/#{name}\s*=\s*["'](?<name>[^"']+)["']/) + match[:name] if match + end + end +end diff --git a/app/models/blob_viewer/download.rb b/app/models/blob_viewer/download.rb new file mode 100644 index 00000000000..074e7204814 --- /dev/null +++ b/app/models/blob_viewer/download.rb @@ -0,0 +1,9 @@ +module BlobViewer + class Download < Base + include Simple + include Static + + self.partial_name = 'download' + self.binary = true + end +end diff --git a/app/models/blob_viewer/empty.rb b/app/models/blob_viewer/empty.rb new file mode 100644 index 00000000000..d9d128eb273 --- /dev/null +++ b/app/models/blob_viewer/empty.rb @@ -0,0 +1,9 @@ +module BlobViewer + class Empty < Base + include Simple + include ServerSide + + self.partial_name = 'empty' + self.binary = true + end +end diff --git a/app/models/blob_viewer/gemfile.rb b/app/models/blob_viewer/gemfile.rb new file mode 100644 index 00000000000..fae8c8df23f --- /dev/null +++ b/app/models/blob_viewer/gemfile.rb @@ -0,0 +1,15 @@ +module BlobViewer + class Gemfile < DependencyManager + include Static + + self.file_types = %i(gemfile gemfile_lock) + + def manager_name + 'Bundler' + end + + def manager_url + 'http://bundler.io/' + end + end +end diff --git a/app/models/blob_viewer/gemspec.rb b/app/models/blob_viewer/gemspec.rb new file mode 100644 index 00000000000..7802edeb754 --- /dev/null +++ b/app/models/blob_viewer/gemspec.rb @@ -0,0 +1,27 @@ +module BlobViewer + class Gemspec < DependencyManager + include ServerSide + + self.file_types = %i(gemspec) + + def manager_name + 'RubyGems' + end + + def manager_url + 'https://rubygems.org/' + end + + def package_type + 'gem' + end + + def package_name + @package_name ||= package_name_from_method_call('name') + end + + def package_url + "https://rubygems.org/gems/#{package_name}" + end + end +end diff --git a/app/models/blob_viewer/gitlab_ci_yml.rb b/app/models/blob_viewer/gitlab_ci_yml.rb new file mode 100644 index 00000000000..7267c3965d3 --- /dev/null +++ b/app/models/blob_viewer/gitlab_ci_yml.rb @@ -0,0 +1,23 @@ +module BlobViewer + class GitlabCiYml < Base + include ServerSide + include Auxiliary + + self.partial_name = 'gitlab_ci_yml' + self.loading_partial_name = 'gitlab_ci_yml_loading' + self.file_types = %i(gitlab_ci) + self.binary = false + + def validation_message + return @validation_message if defined?(@validation_message) + + prepare! + + @validation_message = Ci::GitlabCiYamlProcessor.validation_message(blob.data) + end + + def valid? + validation_message.blank? + end + end +end diff --git a/app/models/blob_viewer/godeps_json.rb b/app/models/blob_viewer/godeps_json.rb new file mode 100644 index 00000000000..e19a602603b --- /dev/null +++ b/app/models/blob_viewer/godeps_json.rb @@ -0,0 +1,15 @@ +module BlobViewer + class GodepsJson < DependencyManager + include Static + + self.file_types = %i(godeps_json) + + def manager_name + 'godep' + end + + def manager_url + 'https://github.com/tools/godep' + end + end +end diff --git a/app/models/blob_viewer/image.rb b/app/models/blob_viewer/image.rb new file mode 100644 index 00000000000..c4eae5c79c2 --- /dev/null +++ b/app/models/blob_viewer/image.rb @@ -0,0 +1,12 @@ +module BlobViewer + class Image < Base + include Rich + include ClientSide + + self.partial_name = 'image' + self.extensions = UploaderHelper::IMAGE_EXT + self.binary = true + self.switcher_icon = 'picture-o' + self.switcher_title = 'image' + end +end diff --git a/app/models/blob_viewer/license.rb b/app/models/blob_viewer/license.rb new file mode 100644 index 00000000000..57355f2c3aa --- /dev/null +++ b/app/models/blob_viewer/license.rb @@ -0,0 +1,20 @@ +module BlobViewer + class License < Base + include Auxiliary + include Static + + self.partial_name = 'license' + self.file_types = %i(license) + self.binary = false + + def license + project.repository.license + end + + def render_error + return if license + + :unknown_license + end + end +end diff --git a/app/models/blob_viewer/markup.rb b/app/models/blob_viewer/markup.rb new file mode 100644 index 00000000000..33b59c4f512 --- /dev/null +++ b/app/models/blob_viewer/markup.rb @@ -0,0 +1,11 @@ +module BlobViewer + class Markup < Base + include Rich + include ServerSide + + self.partial_name = 'markup' + self.extensions = Gitlab::MarkupHelper::EXTENSIONS + self.file_types = %i(readme) + self.binary = false + end +end diff --git a/app/models/blob_viewer/notebook.rb b/app/models/blob_viewer/notebook.rb new file mode 100644 index 00000000000..8632b8a9885 --- /dev/null +++ b/app/models/blob_viewer/notebook.rb @@ -0,0 +1,12 @@ +module BlobViewer + class Notebook < Base + include Rich + include ClientSide + + self.partial_name = 'notebook' + self.extensions = %w(ipynb) + self.binary = false + self.switcher_icon = 'file-text-o' + self.switcher_title = 'notebook' + end +end diff --git a/app/models/blob_viewer/package_json.rb b/app/models/blob_viewer/package_json.rb new file mode 100644 index 00000000000..09221efb56c --- /dev/null +++ b/app/models/blob_viewer/package_json.rb @@ -0,0 +1,23 @@ +module BlobViewer + class PackageJson < DependencyManager + include ServerSide + + self.file_types = %i(package_json) + + def manager_name + 'npm' + end + + def manager_url + 'https://www.npmjs.com/' + end + + def package_name + @package_name ||= package_name_from_json('name') + end + + def package_url + "https://www.npmjs.com/package/#{package_name}" + end + end +end diff --git a/app/models/blob_viewer/pdf.rb b/app/models/blob_viewer/pdf.rb new file mode 100644 index 00000000000..65805f5f388 --- /dev/null +++ b/app/models/blob_viewer/pdf.rb @@ -0,0 +1,12 @@ +module BlobViewer + class PDF < Base + include Rich + include ClientSide + + self.partial_name = 'pdf' + self.extensions = %w(pdf) + self.binary = true + self.switcher_icon = 'file-pdf-o' + self.switcher_title = 'PDF' + end +end diff --git a/app/models/blob_viewer/podfile.rb b/app/models/blob_viewer/podfile.rb new file mode 100644 index 00000000000..507bc734cb4 --- /dev/null +++ b/app/models/blob_viewer/podfile.rb @@ -0,0 +1,15 @@ +module BlobViewer + class Podfile < DependencyManager + include Static + + self.file_types = %i(podfile) + + def manager_name + 'CocoaPods' + end + + def manager_url + 'https://cocoapods.org/' + end + end +end diff --git a/app/models/blob_viewer/podspec.rb b/app/models/blob_viewer/podspec.rb new file mode 100644 index 00000000000..a4c242db3a9 --- /dev/null +++ b/app/models/blob_viewer/podspec.rb @@ -0,0 +1,27 @@ +module BlobViewer + class Podspec < DependencyManager + include ServerSide + + self.file_types = %i(podspec) + + def manager_name + 'CocoaPods' + end + + def manager_url + 'https://cocoapods.org/' + end + + def package_type + 'pod' + end + + def package_name + @package_name ||= package_name_from_method_call('name') + end + + def package_url + "https://cocoapods.org/pods/#{package_name}" + end + end +end diff --git a/app/models/blob_viewer/podspec_json.rb b/app/models/blob_viewer/podspec_json.rb new file mode 100644 index 00000000000..602f4a51fd9 --- /dev/null +++ b/app/models/blob_viewer/podspec_json.rb @@ -0,0 +1,9 @@ +module BlobViewer + class PodspecJson < Podspec + self.file_types = %i(podspec_json) + + def package_name + @package_name ||= package_name_from_json('name') + end + end +end diff --git a/app/models/blob_viewer/readme.rb b/app/models/blob_viewer/readme.rb new file mode 100644 index 00000000000..75c373a03bb --- /dev/null +++ b/app/models/blob_viewer/readme.rb @@ -0,0 +1,14 @@ +module BlobViewer + class Readme < Base + include Auxiliary + include Static + + self.partial_name = 'readme' + self.file_types = %i(readme) + self.binary = false + + def visible_to?(current_user) + can?(current_user, :read_wiki, project) + end + end +end diff --git a/app/models/blob_viewer/requirements_txt.rb b/app/models/blob_viewer/requirements_txt.rb new file mode 100644 index 00000000000..83ac55f61d0 --- /dev/null +++ b/app/models/blob_viewer/requirements_txt.rb @@ -0,0 +1,15 @@ +module BlobViewer + class RequirementsTxt < DependencyManager + include Static + + self.file_types = %i(requirements_txt) + + def manager_name + 'pip' + end + + def manager_url + 'https://pip.pypa.io/' + end + end +end diff --git a/app/models/blob_viewer/rich.rb b/app/models/blob_viewer/rich.rb new file mode 100644 index 00000000000..be373dbc948 --- /dev/null +++ b/app/models/blob_viewer/rich.rb @@ -0,0 +1,11 @@ +module BlobViewer + module Rich + extend ActiveSupport::Concern + + included do + self.type = :rich + self.switcher_icon = 'file-text-o' + self.switcher_title = 'rendered file' + end + end +end diff --git a/app/models/blob_viewer/route_map.rb b/app/models/blob_viewer/route_map.rb new file mode 100644 index 00000000000..153b4eeb2c9 --- /dev/null +++ b/app/models/blob_viewer/route_map.rb @@ -0,0 +1,30 @@ +module BlobViewer + class RouteMap < Base + include ServerSide + include Auxiliary + + self.partial_name = 'route_map' + self.loading_partial_name = 'route_map_loading' + self.file_types = %i(route_map) + self.binary = false + + def validation_message + return @validation_message if defined?(@validation_message) + + prepare! + + @validation_message = + begin + Gitlab::RouteMap.new(blob.data) + + nil + rescue Gitlab::RouteMap::FormatError => e + e.message + end + end + + def valid? + validation_message.blank? + end + end +end diff --git a/app/models/blob_viewer/server_side.rb b/app/models/blob_viewer/server_side.rb new file mode 100644 index 00000000000..87884dcd6bf --- /dev/null +++ b/app/models/blob_viewer/server_side.rb @@ -0,0 +1,30 @@ +module BlobViewer + module ServerSide + extend ActiveSupport::Concern + + included do + self.load_async = true + self.overridable_max_size = 2.megabytes + self.max_size = 5.megabytes + end + + def prepare! + if blob.project + blob.load_all_data!(blob.project.repository) + end + end + + def render_error + if blob.stored_externally? + # Files that are not stored in the repository, like LFS files and + # build artifacts, can only be rendered using a client-side viewer, + # since we do not want to read large amounts of data into memory on the + # server side. Client-side viewers use JS and can fetch the file from + # `blob_raw_url` using AJAX. + return :server_side_but_stored_externally + end + + super + end + end +end diff --git a/app/models/blob_viewer/simple.rb b/app/models/blob_viewer/simple.rb new file mode 100644 index 00000000000..454a20495fc --- /dev/null +++ b/app/models/blob_viewer/simple.rb @@ -0,0 +1,11 @@ +module BlobViewer + module Simple + extend ActiveSupport::Concern + + included do + self.type = :simple + self.switcher_icon = 'code' + self.switcher_title = 'source' + end + end +end diff --git a/app/models/blob_viewer/sketch.rb b/app/models/blob_viewer/sketch.rb new file mode 100644 index 00000000000..818456778e1 --- /dev/null +++ b/app/models/blob_viewer/sketch.rb @@ -0,0 +1,12 @@ +module BlobViewer + class Sketch < Base + include Rich + include ClientSide + + self.partial_name = 'sketch' + self.extensions = %w(sketch) + self.binary = true + self.switcher_icon = 'file-image-o' + self.switcher_title = 'preview' + end +end diff --git a/app/models/blob_viewer/static.rb b/app/models/blob_viewer/static.rb new file mode 100644 index 00000000000..c9e257e5388 --- /dev/null +++ b/app/models/blob_viewer/static.rb @@ -0,0 +1,14 @@ +module BlobViewer + module Static + extend ActiveSupport::Concern + + included do + self.load_async = false + end + + # We can always render a static viewer, even if the blob is too large. + def render_error + nil + end + end +end diff --git a/app/models/blob_viewer/svg.rb b/app/models/blob_viewer/svg.rb new file mode 100644 index 00000000000..b7e5cd71e6b --- /dev/null +++ b/app/models/blob_viewer/svg.rb @@ -0,0 +1,12 @@ +module BlobViewer + class SVG < Base + include Rich + include ServerSide + + self.partial_name = 'svg' + self.extensions = %w(svg) + self.binary = false + self.switcher_icon = 'picture-o' + self.switcher_title = 'image' + end +end diff --git a/app/models/blob_viewer/text.rb b/app/models/blob_viewer/text.rb new file mode 100644 index 00000000000..eddca50b4d4 --- /dev/null +++ b/app/models/blob_viewer/text.rb @@ -0,0 +1,11 @@ +module BlobViewer + class Text < Base + include Simple + include ServerSide + + self.partial_name = 'text' + self.binary = false + self.overridable_max_size = 1.megabyte + self.max_size = 10.megabytes + end +end diff --git a/app/models/blob_viewer/text_stl.rb b/app/models/blob_viewer/text_stl.rb new file mode 100644 index 00000000000..8184dc0104c --- /dev/null +++ b/app/models/blob_viewer/text_stl.rb @@ -0,0 +1,5 @@ +module BlobViewer + class TextSTL < BinarySTL + self.binary = false + end +end diff --git a/app/models/blob_viewer/video.rb b/app/models/blob_viewer/video.rb new file mode 100644 index 00000000000..057f9fe516f --- /dev/null +++ b/app/models/blob_viewer/video.rb @@ -0,0 +1,12 @@ +module BlobViewer + class Video < Base + include Rich + include ClientSide + + self.partial_name = 'video' + self.extensions = UploaderHelper::VIDEO_EXT + self.binary = true + self.switcher_icon = 'film' + self.switcher_title = 'video' + end +end diff --git a/app/models/blob_viewer/yarn_lock.rb b/app/models/blob_viewer/yarn_lock.rb new file mode 100644 index 00000000000..31588ddcbab --- /dev/null +++ b/app/models/blob_viewer/yarn_lock.rb @@ -0,0 +1,15 @@ +module BlobViewer + class YarnLock < DependencyManager + include Static + + self.file_types = %i(yarn_lock) + + def manager_name + 'Yarn' + end + + def manager_url + 'https://yarnpkg.com/' + end + end +end diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb new file mode 100644 index 00000000000..b35febc9ac5 --- /dev/null +++ b/app/models/ci/artifact_blob.rb @@ -0,0 +1,35 @@ +module Ci + class ArtifactBlob + include BlobLike + + attr_reader :entry + + def initialize(entry) + @entry = entry + end + + delegate :name, :path, to: :entry + + def id + Digest::SHA1.hexdigest(path) + end + + def size + entry.metadata[:size] + end + + def data + "Build artifact #{path}" + end + + def mode + entry.metadata[:mode] + end + + def external_storage + :build_artifact + end + + alias_method :external_size, :size + end +end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index dbe4a2bf43f..1581ba9e55d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -103,27 +103,17 @@ module Ci end def playable? - project.builds_enabled? && has_commands? && - action? && manual? + action? && manual? end def action? self.when == 'manual' end - def has_commands? - commands.present? - end - def play(current_user) - # Try to queue a current build - if self.enqueue - self.update(user: current_user) - self - else - # Otherwise we need to create a duplicate - Ci::Build.retry(self, current_user) - end + Ci::PlayBuildService + .new(project, current_user) + .execute(self) end def cancelable? @@ -131,12 +121,11 @@ module Ci end def retryable? - project.builds_enabled? && has_commands? && - (success? || failed? || canceled?) + success? || failed? || canceled? end - def retried? - !self.pipeline.statuses.latest.include?(self) + def latest? + !retried? end def expanded_environment_name @@ -171,19 +160,6 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def trace_html(**args) - trace_with_state(**args)[:html] || '' - end - - def trace_with_state(state: nil, last_lines: nil) - trace_ansi = trace(last_lines: last_lines) - if trace_ansi.present? - Ci::Ansi2html.convert(trace_ansi, state) - else - {} - end - end - def timeout project.build_timeout end @@ -244,136 +220,35 @@ module Ci end def update_coverage - coverage = extract_coverage(trace, coverage_regex) + coverage = trace.extract_coverage(coverage_regex) update_attributes(coverage: coverage) if coverage.present? end - def extract_coverage(text, regex) - return unless regex - - matches = text.scan(Regexp.new(regex)).last - matches = matches.last if matches.is_a?(Array) - coverage = matches.gsub(/\d+(\.\d+)?/).first - - if coverage.present? - coverage.to_f - end - rescue - # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now - end - - def has_trace_file? - File.exist?(path_to_trace) || has_old_trace_file? + def trace + Gitlab::Ci::Trace.new(self) end def has_trace? - raw_trace.present? - end - - def raw_trace(last_lines: nil) - if File.exist?(trace_file_path) - Gitlab::Ci::TraceReader.new(trace_file_path). - read(last_lines: last_lines) - else - # backward compatibility - read_attribute :trace - end - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - def has_old_trace_file? - project.ci_id && File.exist?(old_path_to_trace) - end - - def trace(last_lines: nil) - hide_secrets(raw_trace(last_lines: last_lines)) - end - - def trace_length - if raw_trace - raw_trace.bytesize - else - 0 - end + trace.exist? end - def trace=(trace) - recreate_trace_dir - trace = hide_secrets(trace) - File.write(path_to_trace, trace) + def trace=(data) + raise NotImplementedError end - def recreate_trace_dir - unless Dir.exist?(dir_to_trace) - FileUtils.mkdir_p(dir_to_trace) - end + def old_trace + read_attribute(:trace) end - private :recreate_trace_dir - - def append_trace(trace_part, offset) - recreate_trace_dir - touch if needs_touch? - - trace_part = hide_secrets(trace_part) - File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) - File.open(path_to_trace, 'ab') do |f| - f.write(trace_part) - end + def erase_old_trace! + write_attribute(:trace, nil) + save end def needs_touch? Time.now - updated_at > 15.minutes.to_i end - def trace_file_path - if has_old_trace_file? - old_path_to_trace - else - path_to_trace - end - end - - def dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.id.to_s - ) - end - - def path_to_trace - "#{dir_to_trace}/#{id}.log" - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.ci_id.to_s - ) - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_path_to_trace - "#{old_dir_to_trace}/#{id}.log" - end - ## # Deprecated # @@ -425,8 +300,8 @@ module Ci def execute_hooks return unless project build_data = Gitlab::DataBuilder::Build.build(self) - project.execute_hooks(build_data.dup, :build_hooks) - project.execute_services(build_data.dup, :build_hooks) + project.execute_hooks(build_data.dup, :job_hooks) + project.execute_services(build_data.dup, :job_hooks) PagesService.new(build_data).execute project.running_or_pending_build_count(force: true) end @@ -540,6 +415,8 @@ module Ci end def dependencies + return [] if empty_dependencies? + depended_jobs = depends_on_builds return depended_jobs unless options[:dependencies].present? @@ -549,6 +426,19 @@ module Ci end end + def empty_dependencies? + options[:dependencies]&.empty? + end + + def hide_secrets(trace) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) + trace + end + private def update_artifacts_size @@ -560,7 +450,7 @@ module Ci end def erase_trace! - self.trace = nil + trace.erase! end def update_erased!(user = nil) @@ -622,15 +512,6 @@ module Ci pipeline.config_processor.build_attributes(name) end - def hide_secrets(trace) - return unless trace - - trace = trace.dup - Ci::MaskSecret.mask!(trace, project.runners_token) if project - Ci::MaskSecret.mask!(trace, token) - trace - end - def update_project_statistics return unless project diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb new file mode 100644 index 00000000000..87898b086c6 --- /dev/null +++ b/app/models/ci/group.rb @@ -0,0 +1,40 @@ +module Ci + ## + # This domain model is a representation of a group of jobs that are related + # to each other, like `rspec 0 1`, `rspec 0 2`. + # + # It is not persisted in the database. + # + class Group + include StaticModel + + attr_reader :stage, :name, :jobs + + delegate :size, to: :jobs + + def initialize(stage, name:, jobs:) + @stage = stage + @name = name + @jobs = jobs + end + + def status + @status ||= commit_statuses.status + end + + def detailed_status(current_user) + if jobs.one? + jobs.first.detailed_status(current_user) + else + Gitlab::Ci::Status::Group::Factory + .new(self, current_user).fabricate! + end + end + + private + + def commit_statuses + @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id)) + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e079072a23f..fa1312154ca 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -4,14 +4,30 @@ module Ci include HasStatus include Importable include AfterCommitQueue + include Presentable belongs_to :project belongs_to :user + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' + belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' + + has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' + has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id has_many :builds, foreign_key: :commit_id has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id + # Merge requests for which the current pipeline is running against + # the merge request's latest commit. + has_many :merge_requests, foreign_key: "head_pipeline_id" + + has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build' + has_many :retryable_builds, -> { latest.failed_or_canceled }, foreign_key: :commit_id, class_name: 'Ci::Build' + has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus' + has_many :manual_actions, -> { latest.manual_actions }, foreign_key: :commit_id, class_name: 'Ci::Build' + has_many :artifacts, -> { latest.with_artifacts_not_expired }, foreign_key: :commit_id, class_name: 'Ci::Build' + delegate :id, to: :project, prefix: true validates :sha, presence: { unless: :importing? } @@ -20,7 +36,6 @@ module Ci validate :valid_commit_sha, unless: :importing? after_create :keep_around_commits, unless: :importing? - after_create :refresh_build_status_cache state_machine :status, initial: :created do event :enqueue do @@ -65,23 +80,32 @@ module Ci pipeline.update_duration end + before_transition any => [:manual] do |pipeline| + pipeline.update_duration + end + + before_transition canceled: any - [:canceled] do |pipeline| + pipeline.auto_canceled_by = nil + end + after_transition [:created, :pending] => :running do |pipeline| - pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) } end after_transition any => [:success] do |pipeline| - pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) } + pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) } end after_transition [:created, :pending, :running] => :success do |pipeline| - pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) } + pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) } end after_transition do |pipeline, transition| next if transition.loopback? pipeline.run_after_commit do - PipelineHooksWorker.perform_async(id) + PipelineHooksWorker.perform_async(pipeline.id) + ExpirePipelineCacheWorker.perform_async(pipeline.id) end end @@ -247,11 +271,37 @@ module Ci statuses_with(status: HasStatus::CANCELABLE_STATUSES).any? end + def stuck? + pending_builds.any?(&:stuck?) + end + + def retryable? + retryable_builds.any? + end + + def cancelable? + cancelable_statuses.any? + end + + def auto_canceled? + canceled? && auto_canceled_by_id? + end + def cancel_running - Gitlab::OptimisticLocking.retry_lock( - statuses.cancelable) do |cancelable| - cancelable.find_each(&:cancel) + Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable| + cancelable.find_each do |job| + yield(job) if block_given? + job.cancel end + end + end + + def auto_cancel_running(pipeline) + update(auto_canceled_by: pipeline) + + cancel_running do |job| + job.auto_canceled_by = pipeline + end end def retry_failed(current_user) @@ -359,7 +409,6 @@ module Ci when 'manual' then block end end - refresh_build_status_cache end def predefined_variables @@ -387,12 +436,9 @@ module Ci project.execute_services(data, :pipeline_hooks) end - # Merge requests for which the current pipeline is running against - # the merge request's latest commit. - def merge_requests - @merge_requests ||= project.merge_requests - .where(source_branch: self.ref) - .select { |merge_request| merge_request.head_pipeline.try(:id) == self.id } + # All the merge requests for which the current pipeline runs/ran against + def all_merge_requests + @all_merge_requests ||= project.merge_requests.where(source_branch: ref) end def detailed_status(current_user) @@ -401,10 +447,6 @@ module Ci .fabricate! end - def refresh_build_status_cache - Ci::PipelineStatus.new(project, sha: sha, status: status).store_in_cache_if_needed - end - private def pipeline_data diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb new file mode 100644 index 00000000000..cf6e53c4ca4 --- /dev/null +++ b/app/models/ci/pipeline_schedule.rb @@ -0,0 +1,60 @@ +module Ci + class PipelineSchedule < ActiveRecord::Base + extend Ci::Model + include Importable + + acts_as_paranoid + + belongs_to :project + belongs_to :owner, class_name: 'User' + has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' + has_many :pipelines + + validates :cron, unless: :importing_or_inactive?, cron: true, presence: { unless: :importing_or_inactive? } + validates :cron_timezone, cron_timezone: true, presence: { unless: :importing_or_inactive? } + validates :ref, presence: { unless: :importing_or_inactive? } + validates :description, presence: true + + before_save :set_next_run_at + + scope :active, -> { where(active: true) } + scope :inactive, -> { where(active: false) } + + def owned_by?(current_user) + owner == current_user + end + + def inactive? + !active? + end + + def deactivate! + update_attribute(:active, false) + end + + def importing_or_inactive? + importing? || inactive? + end + + def runnable_by_owner? + Ability.allowed?(owner, :create_pipeline, project) + end + + def set_next_run_at + self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now) + end + + def schedule_next_run! + save! # with set_next_run_at + rescue ActiveRecord::RecordInvalid + update_attribute(:next_run_at, nil) # update without validation + end + + def real_next_run( + worker_cron: Settings.cron_jobs['pipeline_schedule_worker']['cron'], + worker_time_zone: Time.zone.name) + Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) + .next_time_from(next_run_at) + end + end +end diff --git a/app/models/ci/pipeline_status.rb b/app/models/ci/pipeline_status.rb deleted file mode 100644 index 048047d0e34..00000000000 --- a/app/models/ci/pipeline_status.rb +++ /dev/null @@ -1,86 +0,0 @@ -# This class is not backed by a table in the main database. -# It loads the latest Pipeline for the HEAD of a repository, and caches that -# in Redis. -module Ci - class PipelineStatus - attr_accessor :sha, :status, :project, :loaded - - delegate :commit, to: :project - - def self.load_for_project(project) - new(project).tap do |status| - status.load_status - end - end - - def initialize(project, sha: nil, status: nil) - @project = project - @sha = sha - @status = status - end - - def has_status? - loaded? && sha.present? && status.present? - end - - def load_status - return if loaded? - - if has_cache? - load_from_cache - else - load_from_commit - store_in_cache - end - - self.loaded = true - end - - def load_from_commit - return unless commit - - self.sha = commit.sha - self.status = commit.status - end - - # We only cache the status for the HEAD commit of a project - # This status is rendered in project lists - def store_in_cache_if_needed - return unless sha - return delete_from_cache unless commit - store_in_cache if commit.sha == self.sha - end - - def load_from_cache - Gitlab::Redis.with do |redis| - self.sha, self.status = redis.hmget(cache_key, :sha, :status) - end - end - - def store_in_cache - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, { sha: sha, status: status }) - end - end - - def delete_from_cache - Gitlab::Redis.with do |redis| - redis.del(cache_key) - end - end - - def has_cache? - Gitlab::Redis.with do |redis| - redis.exists(cache_key) - end - end - - def loaded? - self.loaded - end - - def cache_key - "projects/#{project.id}/build_status" - end - end -end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index e7d6b17d445..9bda3186c30 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -15,6 +15,14 @@ module Ci @warnings = warnings end + def groups + @groups ||= statuses.ordered.latest + .sort_by(&:sortable_name).group_by(&:group_name) + .map do |group_name, grouped_statuses| + Ci::Group.new(self, name: group_name, jobs: grouped_statuses) + end + end + def to_param name end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index cba1d81a861..6df41a3f301 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -7,7 +7,7 @@ module Ci belongs_to :project belongs_to :owner, class_name: "User" - has_many :trigger_requests, dependent: :destroy + has_many :trigger_requests validates :token, presence: true, uniqueness: true diff --git a/app/models/commit.rb b/app/models/commit.rb index e71f1769255..2b8a6fdd4ab 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -2,6 +2,7 @@ class Commit extend ActiveModel::Naming include ActiveModel::Conversion + include Noteable include Participable include Mentionable include Referable @@ -48,7 +49,7 @@ class Commit def max_diff_options { max_files: DIFF_HARD_LIMIT_FILES, - max_lines: DIFF_HARD_LIMIT_LINES, + max_lines: DIFF_HARD_LIMIT_LINES } end @@ -200,6 +201,10 @@ class Commit project.notes.for_commit_id(self.id) end + def discussion_notes + notes.non_diff_notes + end + def notes_with_associations notes.includes(:author) end @@ -228,8 +233,8 @@ class Commit project.pipelines.where(sha: sha) end - def latest_pipeline - pipelines.last + def last_pipeline + @last_pipeline ||= pipelines.last end def status(ref = nil) @@ -308,7 +313,7 @@ class Commit def uri_type(path) entry = @raw.tree.path(path) if entry[:type] == :blob - blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: entry[:name])) + blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: entry[:name]), @project) blob.image? || blob.video? ? :raw : :blob else entry[:type] @@ -318,16 +323,23 @@ class Commit end def raw_diffs(*args) - use_gitaly = Gitlab::GitalyClient.feature_enabled?(:commit_raw_diffs) - deltas_only = args.last.is_a?(Hash) && args.last[:deltas_only] - - if use_gitaly && !deltas_only - Gitlab::GitalyClient::Commit.diff_from_parent(self, *args) + if Gitlab::GitalyClient.feature_enabled?(:commit_raw_diffs) + Gitlab::GitalyClient::Commit.new(project.repository).diff_from_parent(self, *args) else raw.diffs(*args) end end + def raw_deltas + @deltas ||= Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::Commit.new(project.repository).commit_deltas(self) + else + raw.deltas + end + end + end + def diffs(diff_options = nil) Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options) end @@ -383,7 +395,7 @@ class Commit def repo_changes changes = { added: [], modified: [], removed: [] } - raw_diffs(deltas_only: true).each do |diff| + raw_deltas.each do |diff| if diff.deleted_file changes[:removed] << diff.old_path elsif diff.renamed_file || diff.new_file diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 17b322b5ae3..ffafc678968 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -7,6 +7,7 @@ class CommitStatus < ActiveRecord::Base belongs_to :project belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :user delegate :commit, to: :pipeline @@ -17,13 +18,7 @@ class CommitStatus < ActiveRecord::Base validates :name, presence: true alias_attribute :author, :user - - scope :latest, -> do - max_id = unscope(:select).select("max(#{quoted_table_name}.id)") - - where(id: max_id.group(:name, :commit_id)) - end - + scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) end @@ -36,7 +31,8 @@ class CommitStatus < ActiveRecord::Base false, all_state_names - [:failed, :canceled, :manual]) end - scope :retried, -> { where.not(id: latest) } + scope :latest, -> { where(retried: [false, nil]) } + scope :retried, -> { where(retried: true) } scope :ordered, -> { order(:name) } scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } @@ -137,10 +133,8 @@ class CommitStatus < ActiveRecord::Base false end - # Added in 9.0 to keep backward compatibility for projects exported in 8.17 - # and prior. - def gl_project_id - 'dummy' + def auto_canceled? + canceled? && auto_canceled_by_id? end def detailed_status(current_user) diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb new file mode 100644 index 00000000000..8fbfed11bdf --- /dev/null +++ b/app/models/concerns/avatarable.rb @@ -0,0 +1,18 @@ +module Avatarable + extend ActiveSupport::Concern + + def avatar_path(only_path: true) + return unless self[:avatar].present? + + # If only_path is true then use the relative path of avatar. + # Otherwise use full path (including host). + asset_host = ActionController::Base.asset_host + gitlab_host = only_path ? gitlab_config.relative_url_root : gitlab_config.url + + # If asset_host is set then it is expected that assets are handled by a standalone host. + # That means we do not want to get GitLab's relative_url_root option anymore. + host = asset_host.present? ? asset_host : gitlab_host + + [host, avatar.url].join + end +end diff --git a/app/models/concerns/blob_like.rb b/app/models/concerns/blob_like.rb new file mode 100644 index 00000000000..adb81561000 --- /dev/null +++ b/app/models/concerns/blob_like.rb @@ -0,0 +1,48 @@ +module BlobLike + extend ActiveSupport::Concern + include Linguist::BlobHelper + + def id + raise NotImplementedError + end + + def name + raise NotImplementedError + end + + def path + raise NotImplementedError + end + + def size + 0 + end + + def data + nil + end + + def mode + nil + end + + def binary? + false + end + + def load_all_data!(repository) + # No-op + end + + def truncated? + false + end + + def external_storage + nil + end + + def external_size + nil + end +end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 8ea95beed79..eb32bf3d32a 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -8,6 +8,14 @@ # # Corresponding foo_html, bar_html and baz_html fields should exist. module CacheMarkdownField + extend ActiveSupport::Concern + + # Increment this number every time the renderer changes its output + CACHE_VERSION = 1 + + # changes to these attributes cause the cache to be invalidates + INVALIDATED_BY = %w[author project].freeze + # Knows about the relationship between markdown and html field names, and # stores the rendering contexts for the latter class FieldData @@ -30,60 +38,74 @@ module CacheMarkdownField end end - # Dynamic registries don't really work in Rails as it's not guaranteed that - # every class will be loaded, so hardcode the list. - CACHING_CLASSES = %w[ - AbuseReport - Appearance - ApplicationSetting - BroadcastMessage - Issue - Label - MergeRequest - Milestone - Namespace - Note - Project - Release - Snippet - ].freeze - - def self.caching_classes - CACHING_CLASSES.map(&:constantize) - end - def skip_project_check? false end - extend ActiveSupport::Concern + # Returns the default Banzai render context for the cached markdown field. + def banzai_render_context(field) + raise ArgumentError.new("Unknown field: #{field.inspect}") unless + cached_markdown_fields.markdown_fields.include?(field) - included do - cattr_reader :cached_markdown_fields do - FieldData.new - end + # Always include a project key, or Banzai complains + project = self.project if self.respond_to?(:project) + context = cached_markdown_fields[field].merge(project: project) - # Returns the default Banzai render context for the cached markdown field. - def banzai_render_context(field) - raise ArgumentError.new("Unknown field: #{field.inspect}") unless - cached_markdown_fields.markdown_fields.include?(field) + # Banzai is less strict about authors, so don't always have an author key + context[:author] = self.author if self.respond_to?(:author) - # Always include a project key, or Banzai complains - project = self.project if self.respond_to?(:project) - context = cached_markdown_fields[field].merge(project: project) + context + end - # Banzai is less strict about authors, so don't always have an author key - context[:author] = self.author if self.respond_to?(:author) + # Update every column in a row if any one is invalidated, as we only store + # one version per row + def refresh_markdown_cache!(do_update: false) + options = { skip_project_check: skip_project_check? } - context - end + updates = cached_markdown_fields.markdown_fields.map do |markdown_field| + [ + cached_markdown_fields.html_field(markdown_field), + Banzai::Renderer.cacheless_render_field(self, markdown_field, options) + ] + end.to_h + updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION + + updates.each {|html_field, data| write_attribute(html_field, data) } + + update_columns(updates) if persisted? && do_update + end + + def cached_html_up_to_date?(markdown_field) + html_field = cached_markdown_fields.html_field(markdown_field) + + cached = !cached_html_for(markdown_field).nil? && !__send__(markdown_field).nil? + return false unless cached - # Allow callers to look up the cache field name, rather than hardcoding it - def markdown_cache_field_for(field) - raise ArgumentError.new("Unknown field: #{field}") unless - cached_markdown_fields.markdown_fields.include?(field) + markdown_changed = attribute_changed?(markdown_field) || false + html_changed = attribute_changed?(html_field) || false - cached_markdown_fields.html_field(field) + CacheMarkdownField::CACHE_VERSION == cached_markdown_version && + (html_changed || markdown_changed == html_changed) + end + + def invalidated_markdown_cache? + cached_markdown_fields.html_fields.any? {|html_field| attribute_invalidated?(html_field) } + end + + def attribute_invalidated?(attr) + __send__("#{attr}_invalidated?") + end + + def cached_html_for(markdown_field) + raise ArgumentError.new("Unknown field: #{field}") unless + cached_markdown_fields.markdown_fields.include?(markdown_field) + + __send__(cached_markdown_fields.html_field(markdown_field)) + end + + included do + cattr_reader :cached_markdown_fields do + FieldData.new end # Always exclude _html fields from attributes (including serialization). @@ -92,12 +114,18 @@ module CacheMarkdownField def attributes attrs = attributes_before_markdown_cache + attrs.delete('cached_markdown_version') + cached_markdown_fields.html_fields.each do |field| attrs.delete(field) end attrs end + + # Using before_update here conflicts with elasticsearch-model somehow + before_create :refresh_markdown_cache!, if: :invalidated_markdown_cache? + before_update :refresh_markdown_cache!, if: :invalidated_markdown_cache? end class_methods do @@ -107,31 +135,18 @@ module CacheMarkdownField # a corresponding _html field. Any custom rendering options may be provided # as a context. def cache_markdown_field(markdown_field, context = {}) - raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless - CacheMarkdownField::CACHING_CLASSES.include?(self.to_s) - cached_markdown_fields[markdown_field] = context html_field = cached_markdown_fields.html_field(markdown_field) - cache_method = "#{markdown_field}_cache_refresh".to_sym invalidation_method = "#{html_field}_invalidated?".to_sym - define_method(cache_method) do - options = { skip_project_check: skip_project_check? } - html = Banzai::Renderer.cacheless_render_field(self, markdown_field, options) - __send__("#{html_field}=", html) - true - end - # The HTML becomes invalid if any dependent fields change. For now, assume # author and project invalidate the cache in all circumstances. define_method(invalidation_method) do changed_fields = changed_attributes.keys - invalidations = changed_fields & [markdown_field.to_s, "author", "project"] - !invalidations.empty? + invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] + !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end - - before_save cache_method, if: invalidation_method end end end diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb new file mode 100644 index 00000000000..a7bdf5587b2 --- /dev/null +++ b/app/models/concerns/discussion_on_diff.rb @@ -0,0 +1,50 @@ +# Contains functionality shared between `DiffDiscussion` and `LegacyDiffDiscussion`. +module DiscussionOnDiff + extend ActiveSupport::Concern + + NUMBER_OF_TRUNCATED_DIFF_LINES = 16 + + included do + delegate :line_code, + :original_line_code, + :diff_file, + :diff_line, + :for_line?, + :active?, + :created_at_diff?, + + to: :first_note + + delegate :file_path, + :blob, + :highlighted_diff_lines, + :diff_lines, + + to: :diff_file, + allow_nil: true + end + + def diff_discussion? + true + end + + # Returns an array of at most 16 highlighted lines above a diff note + def truncated_diff_lines(highlight: true) + lines = highlight ? highlighted_diff_lines : diff_lines + prev_lines = [] + + lines.each do |line| + if line.meta? + prev_lines.clear + else + prev_lines << line + + break if for_line?(line) + + prev_lines.shift if prev_lines.length >= NUMBER_OF_TRUNCATED_DIFF_LINES + end + end + + prev_lines + end +end diff --git a/app/models/concerns/ghost_user.rb b/app/models/concerns/ghost_user.rb new file mode 100644 index 00000000000..da696127a80 --- /dev/null +++ b/app/models/concerns/ghost_user.rb @@ -0,0 +1,7 @@ +module GhostUser + extend ActiveSupport::Concern + + def ghost_user? + user && user.ghost? + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index f5f5e64bcbe..ebfffe82510 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -8,7 +8,7 @@ module HasStatus ACTIVE_STATUSES = %w[pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze - CANCELABLE_STATUSES = %w[running pending created manual].freeze + CANCELABLE_STATUSES = %w[running pending created].freeze class_methods do def status_sql @@ -69,7 +69,7 @@ module HasStatus end scope :created, -> { where(status: 'created') } - scope :relevant, -> { where.not(status: 'created') } + scope :relevant, -> { where(status: AVAILABLE_STATUSES - ['created']) } scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } scope :success, -> { where(status: 'success') } @@ -77,6 +77,7 @@ module HasStatus scope :canceled, -> { where(status: 'canceled') } scope :skipped, -> { where(status: 'skipped') } scope :manual, -> { where(status: 'manual') } + scope :created_or_pending, -> { where(status: [:created, :pending]) } scope :running_or_pending, -> { where(status: [:running, :pending]) } scope :finished, -> { where(status: [:success, :failed, :canceled]) } scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } diff --git a/app/models/concerns/ignorable_column.rb b/app/models/concerns/ignorable_column.rb new file mode 100644 index 00000000000..eb9f3423e48 --- /dev/null +++ b/app/models/concerns/ignorable_column.rb @@ -0,0 +1,28 @@ +# Module that can be included into a model to make it easier to ignore database +# columns. +# +# Example: +# +# class User < ActiveRecord::Base +# include IgnorableColumn +# +# ignore_column :updated_at +# end +# +module IgnorableColumn + extend ActiveSupport::Concern + + module ClassMethods + def columns + super.reject { |column| ignored_columns.include?(column.name) } + end + + def ignored_columns + @ignored_columns ||= Set.new + end + + def ignore_column(name) + ignored_columns << name.to_s + end + end +end diff --git a/app/models/concerns/importable.rb b/app/models/concerns/importable.rb index 019ef755849..c9331eaf4cc 100644 --- a/app/models/concerns/importable.rb +++ b/app/models/concerns/importable.rb @@ -3,4 +3,7 @@ module Importable attr_accessor :importing alias_method :importing?, :importing + + attr_accessor :imported + alias_method :imported?, :imported end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 4d54426b79e..075ec575f9d 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -14,6 +14,7 @@ module Issuable include Awardable include Taskable include TimeTrackable + include Importable # This object is used to gather issuable meta data for displaying # upvotes, downvotes, notes and closing merge requests count for issues and merge requests @@ -22,11 +23,11 @@ module Issuable included do cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description + cache_markdown_field :description, issuable_state_filter_enabled: true belongs_to :author, class_name: "User" - belongs_to :assignee, class_name: "User" belongs_to :updated_by, class_name: "User" + belongs_to :last_edited_by, class_name: 'User' belongs_to :milestone has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :destroy do def authors_loaded? @@ -64,11 +65,8 @@ module Issuable validates :title, presence: true, length: { maximum: 255 } scope :authored, ->(user) { where(author_id: user) } - scope :assigned_to, ->(u) { where(assignee_id: u.id)} scope :recent, -> { reorder(id: :desc) } scope :order_position_asc, -> { reorder(position: :asc) } - scope :assigned, -> { where("assignee_id IS NOT NULL") } - scope :unassigned, -> { where("assignee_id IS NULL") } scope :of_projects, ->(ids) { where(project_id: ids) } scope :of_milestones, ->(ids) { where(milestone_id: ids) } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } @@ -91,22 +89,13 @@ module Issuable attr_mentionable :description participant :author - participant :assignee participant :notes_with_associations strip_attributes :title acts_as_paranoid - after_save :update_assignee_cache_counts, if: :assignee_id_changed? - after_save :record_metrics - - def update_assignee_cache_counts - # make sure we flush the cache for both the old *and* new assignees(if they exist) - previous_assignee = User.find_by_id(assignee_id_was) if assignee_id_was - previous_assignee&.update_cache_counts - assignee&.update_cache_counts - end + after_save :record_metrics, unless: :imported? # We want to use optimistic lock for cases when only title or description are involved # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html @@ -236,10 +225,6 @@ module Issuable today? && created_at == updated_at end - def is_being_reassigned? - assignee_id_changed? - end - def open? opened? || reopened? end @@ -268,7 +253,11 @@ module Issuable # DEPRECATED repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } - hook_data[:assignee] = assignee.hook_attrs if assignee + if self.is_a?(Issue) + hook_data[:assignees] = assignees.map(&:hook_attrs) if assignees.any? + else + hook_data[:assignee] = assignee.hook_attrs if assignee + end hook_data end @@ -291,17 +280,6 @@ module Issuable self.class.to_ability_name end - # Convert this Issuable class name to a format usable by notifications. - # - # Examples: - # - # issuable.class # => MergeRequest - # issuable.human_class_name # => "merge request" - - def human_class_name - @human_class_name ||= self.class.name.titleize.downcase - end - # Returns a Hash of attributes to be used for Twitter card metadata def card_attributes { @@ -341,11 +319,6 @@ module Issuable false end - def assignee_or_author?(user) - # We're comparing IDs here so we don't need to load any associations. - author_id == user.id || assignee_id == user.id - end - def record_metrics metrics = self.metrics || create_metrics metrics.record! diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 7e56e371b27..c034bf9cbc0 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -44,14 +44,15 @@ module Mentionable end def all_references(current_user = nil, extractor: nil) + @extractors ||= {} + # Use custom extractor if it's passed in the function parameters. if extractor - @extractor = extractor + @extractors[current_user] = extractor else - @extractor ||= Gitlab::ReferenceExtractor. - new(project, current_user) + extractor = @extractors[current_user] ||= Gitlab::ReferenceExtractor.new(project, current_user) - @extractor.reset_memoized_values + extractor.reset_memoized_values end self.class.mentionable_attrs.each do |attr, options| @@ -62,10 +63,10 @@ module Mentionable skip_project_check: skip_project_check? ) - @extractor.analyze(text, options) + extractor.analyze(text, options) end - @extractor + extractor end def mentioned_users(current_user = nil) @@ -78,6 +79,8 @@ module Mentionable # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. def referenced_mentionables(current_user = self.author) + return [] unless matches_cross_reference_regex? + refs = all_references(current_user) refs = (refs.issues + refs.merge_requests + refs.commits) @@ -87,6 +90,20 @@ module Mentionable refs.reject { |ref| ref == local_reference } end + # Uses regex to quickly determine if mentionables might be referenced + # Allows heavy processing to be skipped + def matches_cross_reference_regex? + reference_pattern = if !project || project.default_issues_tracker? + ReferenceRegexes::DEFAULT_PATTERN + else + ReferenceRegexes::EXTERNAL_PATTERN + end + + self.class.mentionable_attrs.any? do |attr, _| + __send__(attr) =~ reference_pattern + end + end + # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+. def create_cross_references!(author = self.author, without = []) refs = referenced_mentionables(author) diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb new file mode 100644 index 00000000000..1848230ec7e --- /dev/null +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -0,0 +1,22 @@ +module Mentionable + module ReferenceRegexes + def self.reference_pattern(link_patterns, issue_pattern) + Regexp.union(link_patterns, + issue_pattern, + Commit.reference_pattern, + MergeRequest.reference_pattern) + end + + DEFAULT_PATTERN = begin + issue_pattern = Issue.reference_pattern + link_patterns = Regexp.union([Issue, Commit, MergeRequest].map(&:link_reference_pattern)) + reference_pattern(link_patterns, issue_pattern) + end + + EXTERNAL_PATTERN = begin + issue_pattern = ExternalIssue.reference_pattern + link_patterns = URI.regexp(%w(http https)) + reference_pattern(link_patterns, issue_pattern) + end + end +end diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index f449229864d..a3472af5c55 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -40,7 +40,7 @@ module Milestoneish def issues_visible_to_user(user) memoize_per_user(user, :issues_visible_to_user) do IssuesFinder.new(user, issues_finder_params) - .execute.where(milestone_id: milestoneish_ids) + .execute.includes(:assignees).where(milestone_id: milestoneish_ids) end end diff --git a/app/models/concerns/note_on_diff.rb b/app/models/concerns/note_on_diff.rb index b8dd27a7afe..6359f7596b1 100644 --- a/app/models/concerns/note_on_diff.rb +++ b/app/models/concerns/note_on_diff.rb @@ -1,3 +1,4 @@ +# Contains functionality shared between `DiffNote` and `LegacyDiffNote`. module NoteOnDiff extend ActiveSupport::Concern @@ -25,11 +26,21 @@ module NoteOnDiff raise NotImplementedError end - def can_be_award_emoji? + def active?(diff_refs = nil) + raise NotImplementedError + end + + def created_at_diff?(diff_refs) false end - def to_discussion - Discussion.new([self]) + private + + def noteable_diff_refs + if noteable.respond_to?(:diff_sha_refs) + noteable.diff_sha_refs + else + noteable.diff_refs + end end end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb new file mode 100644 index 00000000000..dd1e6630642 --- /dev/null +++ b/app/models/concerns/noteable.rb @@ -0,0 +1,68 @@ +module Noteable + # Names of all implementers of `Noteable` that support resolvable notes. + RESOLVABLE_TYPES = %w(MergeRequest).freeze + + def base_class_name + self.class.base_class.name + end + + # Convert this Noteable class name to a format usable by notifications. + # + # Examples: + # + # noteable.class # => MergeRequest + # noteable.human_class_name # => "merge request" + def human_class_name + @human_class_name ||= base_class_name.titleize.downcase + end + + def supports_resolvable_notes? + RESOLVABLE_TYPES.include?(base_class_name) + end + + def supports_discussions? + DiscussionNote::NOTEABLE_TYPES.include?(base_class_name) + end + + def discussion_notes + notes + end + + delegate :find_discussion, to: :discussion_notes + + def discussions + @discussions ||= discussion_notes + .inc_relations_for_view + .discussions(self) + end + + def grouped_diff_discussions(*args) + # Doesn't use `discussion_notes`, because this may include commit diff notes + # besides MR diff notes, that we do no want to display on the MR Changes tab. + notes.inc_relations_for_view.grouped_diff_discussions(*args) + end + + def resolvable_discussions + @resolvable_discussions ||= discussion_notes.resolvable.discussions(self) + end + + def discussions_resolvable? + resolvable_discussions.any?(&:resolvable?) + end + + def discussions_resolved? + discussions_resolvable? && resolvable_discussions.none?(&:to_be_resolved?) + end + + def discussions_to_be_resolved? + discussions_resolvable? && !discussions_resolved? + end + + def discussions_to_be_resolved + @discussions_to_be_resolved ||= resolvable_discussions.select(&:to_be_resolved?) + end + + def discussions_can_be_resolved_by?(user) + discussions_to_be_resolved.all? { |discussion| discussion.can_resolve?(user) } + end +end diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index 9dd4d9c6f24..a40148a4394 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -2,20 +2,32 @@ module ProtectedBranchAccess extend ActiveSupport::Concern included do + include ProtectedRefAccess + belongs_to :protected_branch + delegate :project, to: :protected_branch - scope :master, -> { where(access_level: Gitlab::Access::MASTER) } - scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } - end + validates :access_level, presence: true, inclusion: { + in: [ + Gitlab::Access::MASTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::NO_ACCESS + ] + } - def humanize - self.class.human_access_levels[self.access_level] - end + def self.human_access_levels + { + Gitlab::Access::MASTER => "Masters", + Gitlab::Access::DEVELOPER => "Developers + Masters", + Gitlab::Access::NO_ACCESS => "No one" + }.with_indifferent_access + end - def check_access(user) - return true if user.is_admin? + def check_access(user) + return false if access_level == Gitlab::Access::NO_ACCESS - project.team.max_member_access(user.id) >= access_level + super + end end end diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb new file mode 100644 index 00000000000..62eaec2407f --- /dev/null +++ b/app/models/concerns/protected_ref.rb @@ -0,0 +1,42 @@ +module ProtectedRef + extend ActiveSupport::Concern + + included do + belongs_to :project + + validates :name, presence: true + validates :project, presence: true + + delegate :matching, :matches?, :wildcard?, to: :ref_matcher + + def self.protected_ref_accessible_to?(ref, user, action:) + access_levels_for_ref(ref, action: action).any? do |access_level| + access_level.check_access(user) + end + end + + def self.developers_can?(action, ref) + access_levels_for_ref(ref, action: action).any? do |access_level| + access_level.access_level == Gitlab::Access::DEVELOPER + end + end + + def self.access_levels_for_ref(ref, action:) + self.matching(ref).map(&:"#{action}_access_levels").flatten + end + + def self.matching(ref_name, protected_refs: nil) + ProtectedRefMatcher.matching(self, ref_name, protected_refs: protected_refs) + end + end + + def commit + project.commit(self.name) + end + + private + + def ref_matcher + @ref_matcher ||= ProtectedRefMatcher.new(self) + end +end diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb new file mode 100644 index 00000000000..c4f158e569a --- /dev/null +++ b/app/models/concerns/protected_ref_access.rb @@ -0,0 +1,18 @@ +module ProtectedRefAccess + extend ActiveSupport::Concern + + included do + scope :master, -> { where(access_level: Gitlab::Access::MASTER) } + scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } + end + + def humanize + self.class.human_access_levels[self.access_level] + end + + def check_access(user) + return true if user.admin? + + project.team.max_member_access(user.id) >= access_level + end +end diff --git a/app/models/concerns/protected_tag_access.rb b/app/models/concerns/protected_tag_access.rb new file mode 100644 index 00000000000..ee65de24dd8 --- /dev/null +++ b/app/models/concerns/protected_tag_access.rb @@ -0,0 +1,11 @@ +module ProtectedTagAccess + extend ActiveSupport::Concern + + included do + include ProtectedRefAccess + + belongs_to :protected_tag + + delegate :project, to: :protected_tag + end +end diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb new file mode 100644 index 00000000000..fed336c29d6 --- /dev/null +++ b/app/models/concerns/repository_mirroring.rb @@ -0,0 +1,17 @@ +module RepositoryMirroring + def set_remote_as_mirror(name) + config = raw_repository.rugged.config + + # This is used to define repository as equivalent as "git clone --mirror" + config["remote.#{name}.fetch"] = 'refs/*:refs/*' + config["remote.#{name}.mirror"] = true + config["remote.#{name}.prune"] = true + end + + def fetch_mirror(remote, url) + add_remote(remote, url) + set_remote_as_mirror(remote) + fetch_remote(remote, forced: true) + remove_remote(remote) + end +end diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb new file mode 100644 index 00000000000..dd979e7bb17 --- /dev/null +++ b/app/models/concerns/resolvable_discussion.rb @@ -0,0 +1,103 @@ +module ResolvableDiscussion + extend ActiveSupport::Concern + + included do + # A number of properties of this `Discussion`, like `first_note` and `resolvable?`, are memoized. + # When this discussion is resolved or unresolved, the values of these properties potentially change. + # To make sure all memoized values are reset when this happens, `update` resets all instance variables with names in + # `memoized_variables`. If you add a memoized method in `ResolvableDiscussion` or any `Discussion` subclass, + # please make sure the instance variable name is added to `memoized_values`, like below. + cattr_accessor :memoized_values, instance_accessor: false do + [] + end + + memoized_values.push( + :resolvable, + :resolved, + :first_note, + :first_note_to_resolve, + :last_resolved_note, + :last_note + ) + + delegate :potentially_resolvable?, to: :first_note + + delegate :resolved_at, + :resolved_by, + + to: :last_resolved_note, + allow_nil: true + end + + def resolvable? + return @resolvable if @resolvable.present? + + @resolvable = potentially_resolvable? && notes.any?(&:resolvable?) + end + + def resolved? + return @resolved if @resolved.present? + + @resolved = resolvable? && notes.none?(&:to_be_resolved?) + end + + def first_note + @first_note ||= notes.first + end + + def first_note_to_resolve + return unless resolvable? + + @first_note_to_resolve ||= notes.find(&:to_be_resolved?) + end + + def last_resolved_note + return unless resolved? + + @last_resolved_note ||= resolved_notes.sort_by(&:resolved_at).last + end + + def resolved_notes + notes.select(&:resolved?) + end + + def to_be_resolved? + resolvable? && !resolved? + end + + def can_resolve?(current_user) + return false unless current_user + return false unless resolvable? + + current_user == self.noteable.author || + current_user.can?(:resolve_note, self.project) + end + + def resolve!(current_user) + return unless resolvable? + + update { |notes| notes.resolve!(current_user) } + end + + def unresolve! + return unless resolvable? + + update { |notes| notes.unresolve! } + end + + private + + def update + # Do not select `Note.resolvable`, so that system notes remain in the collection + notes_relation = Note.where(id: notes.map(&:id)) + + yield(notes_relation) + + # Set the notes array to the updated notes + @notes = notes_relation.fresh.to_a + + self.class.memoized_values.each do |var| + instance_variable_set(:"@#{var}", nil) + end + end +end diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb new file mode 100644 index 00000000000..05eb6f86704 --- /dev/null +++ b/app/models/concerns/resolvable_note.rb @@ -0,0 +1,72 @@ +module ResolvableNote + extend ActiveSupport::Concern + + # Names of all subclasses of `Note` that can be resolvable. + RESOLVABLE_TYPES = %w(DiffNote DiscussionNote).freeze + + included do + belongs_to :resolved_by, class_name: "User" + + validates :resolved_by, presence: true, if: :resolved? + + # Keep this scope in sync with `#potentially_resolvable?` + scope :potentially_resolvable, -> { where(type: RESOLVABLE_TYPES).where(noteable_type: Noteable::RESOLVABLE_TYPES) } + # Keep this scope in sync with `#resolvable?` + scope :resolvable, -> { potentially_resolvable.user } + + scope :resolved, -> { resolvable.where.not(resolved_at: nil) } + scope :unresolved, -> { resolvable.where(resolved_at: nil) } + end + + module ClassMethods + # This method must be kept in sync with `#resolve!` + def resolve!(current_user) + unresolved.update_all(resolved_at: Time.now, resolved_by_id: current_user.id) + end + + # This method must be kept in sync with `#unresolve!` + def unresolve! + resolved.update_all(resolved_at: nil, resolved_by_id: nil) + end + end + + # Keep this method in sync with the `potentially_resolvable` scope + def potentially_resolvable? + RESOLVABLE_TYPES.include?(self.class.name) && noteable.supports_resolvable_notes? + end + + # Keep this method in sync with the `resolvable` scope + def resolvable? + potentially_resolvable? && !system? + end + + def resolved? + return false unless resolvable? + + self.resolved_at.present? + end + + def to_be_resolved? + resolvable? && !resolved? + end + + # If you update this method remember to also update `.resolve!` + def resolve!(current_user) + return unless resolvable? + return if resolved? + + self.resolved_at = Time.now + self.resolved_by = current_user + save! + end + + # If you update this method remember to also update `.unresolve!` + def unresolve! + return unless resolvable? + return unless resolved? + + self.resolved_at = nil + self.resolved_by = nil + save! + end +end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 529fb5ce988..c4463abdfe6 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -5,6 +5,7 @@ module Routable included do has_one :route, as: :source, autosave: true, dependent: :destroy + has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy validates_associated :route validates :route, presence: true @@ -26,16 +27,31 @@ module Routable # Klass.find_by_full_path('gitlab-org/gitlab-ce') # # Returns a single object, or nil. - def find_by_full_path(path) + def find_by_full_path(path, follow_redirects: false) # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so # any literal matches come first, for this we have to use "BINARY". # Without this there's still no guarantee in what order MySQL will return # rows. + # + # Why do we do this? + # + # Even though we have Rails validation on Route for unique paths + # (case-insensitive), there are old projects in our DB (and possibly + # clients' DBs) that have the same path with different cases. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/18603. Also note that + # our unique index is case-sensitive in Postgres. binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" - - where_full_path_in([path]).reorder(order_sql).take + found = where_full_path_in([path]).reorder(order_sql).take + return found if found + + if follow_redirects + if Gitlab::Database.postgresql? + joins(:redirect_routes).find_by("LOWER(redirect_routes.path) = LOWER(?)", path) + else + joins(:redirect_routes).find_by(redirect_routes: { path: path }) + end + end end # Builds a relation to find multiple objects by their full paths. @@ -83,6 +99,74 @@ module Routable AND members.source_type = r2.source_type"). where('members.user_id = ?', user_id) end + + # Builds a relation to find multiple objects that are nested under user + # membership. Includes the parent, as opposed to `#member_descendants` + # which only includes the descendants. + # + # Usage: + # + # Klass.member_self_and_descendants(1) + # + # Returns an ActiveRecord::Relation. + def member_self_and_descendants(user_id) + joins(:route). + joins("INNER JOIN routes r2 ON routes.path LIKE CONCAT(r2.path, '/%') + OR routes.path = r2.path + INNER JOIN members ON members.source_id = r2.source_id + AND members.source_type = r2.source_type"). + where('members.user_id = ?', user_id) + end + + # Returns all objects in a hierarchy, where any node in the hierarchy is + # under the user membership. + # + # Usage: + # + # Klass.member_hierarchy(1) + # + # Examples: + # + # Given the following group tree... + # + # _______group_1_______ + # | | + # | | + # nested_group_1 nested_group_2 + # | | + # | | + # nested_group_1_1 nested_group_2_1 + # + # + # ... the following results are returned: + # + # * the user is a member of group 1 + # => 'group_1', + # 'nested_group_1', nested_group_1_1', + # 'nested_group_2', 'nested_group_2_1' + # + # * the user is a member of nested_group_2 + # => 'group1', + # 'nested_group_2', 'nested_group_2_1' + # + # * the user is a member of nested_group_2_1 + # => 'group1', + # 'nested_group_2', 'nested_group_2_1' + # + # Returns an ActiveRecord::Relation. + def member_hierarchy(user_id) + paths = member_self_and_descendants(user_id).pluck('routes.path') + + return none if paths.empty? + + wheres = paths.map do |path| + "#{connection.quote(path)} = routes.path + OR + #{connection.quote(path)} LIKE CONCAT(routes.path, '/%')" + end + + joins(:route).where(wheres.join(' OR ')) + end end def full_name @@ -95,7 +179,20 @@ module Routable end end + # Every time `project.namespace.becomes(Namespace)` is called for polymorphic_path, + # a new instance is instantiated, and we end up duplicating the same query to retrieve + # the route. Caching this per request ensures that even if we have multiple instances, + # we will not have to duplicate work, avoiding N+1 queries in some cases. def full_path + return uncached_full_path unless RequestStore.active? + + key = "routable/full_path/#{self.class.name}/#{self.id}" + RequestStore[key] ||= uncached_full_path + end + + private + + def uncached_full_path if route && route.path.present? @full_path ||= route.path else @@ -105,8 +202,6 @@ module Routable end end - private - def full_name_changed? name_changed? || parent_changed? end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb new file mode 100644 index 00000000000..d0c94d3b694 --- /dev/null +++ b/app/models/container_repository.rb @@ -0,0 +1,82 @@ +class ContainerRepository < ActiveRecord::Base + belongs_to :project + + validates :name, length: { minimum: 0, allow_nil: false } + validates :name, uniqueness: { scope: :project_id } + + delegate :client, to: :registry + + before_destroy :delete_tags! + + def registry + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(path) + + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end + + def path + @path ||= [project.full_path, name] + .select(&:present?).join('/').downcase + end + + def location + File.join(registry.path, path) + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + @manifest ||= client.repository_tags(path) + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def has_tags? + tags.any? + end + + def root_repository? + name.empty? + end + + def delete_tags! + return unless has_tags? + + digests = tags.map { |tag| tag.digest }.to_set + + digests.all? do |digest| + client.delete_repository_tag(self.path, digest) + end + end + + def self.build_from_path(path) + self.new(project: path.repository_project, + name: path.repository_name) + end + + def self.create_from_path!(path) + build_from_path(path).tap(&:save!) + end + + def self.build_root_repository(project) + self.new(project: project, name: '') + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index afad001d50f..216cec751e3 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -85,8 +85,8 @@ class Deployment < ActiveRecord::Base end def stop_action - return nil unless on_stop.present? - return nil unless manual_actions + return unless on_stop.present? + return unless manual_actions @stop_action ||= manual_actions.find_by(name: on_stop) end @@ -99,6 +99,16 @@ class Deployment < ActiveRecord::Base created_at.to_time.in_time_zone.to_s(:medium) end + def has_metrics? + project.monitoring_service.present? + end + + def metrics + return {} unless has_metrics? + + project.monitoring_service.deployment_metrics(self) + end + private def ref_path diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb new file mode 100644 index 00000000000..14ddd2fcc88 --- /dev/null +++ b/app/models/diff_discussion.rb @@ -0,0 +1,45 @@ +# A discussion on merge request or commit diffs consisting of `DiffNote` notes. +# +# A discussion of this type can be resolvable. +class DiffDiscussion < Discussion + include DiscussionOnDiff + + def self.note_class + DiffNote + end + + delegate :position, + :original_position, + + to: :first_note + + def legacy_diff_discussion? + false + end + + def merge_request_version_params + return unless for_merge_request? + + if active? + {} + else + diff_refs = position.diff_refs + + if diff = noteable.merge_request_diff_for(diff_refs) + { diff_id: diff.id } + elsif diff = noteable.merge_request_diff_for(diff_refs.head_sha) + { + diff_id: diff.id, + start_sha: diff_refs.start_sha + } + end + end + end + + def reply_attributes + super.merge( + original_position: original_position.to_json, + position: position.to_json + ) + end +end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 895a91139c9..76c59199afd 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -1,6 +1,11 @@ +# A note on merge request or commit diffs +# +# A note of this type can be resolvable. class DiffNote < Note include NoteOnDiff + NOTEABLE_TYPES = %w(MergeRequest Commit).freeze + serialize :original_position, Gitlab::Diff::Position serialize :position, Gitlab::Diff::Position @@ -8,59 +13,31 @@ class DiffNote < Note validates :position, presence: true validates :diff_line, presence: true validates :line_code, presence: true, line_code: true - validates :noteable_type, inclusion: { in: %w(Commit MergeRequest) } - validates :resolved_by, presence: true, if: :resolved? + validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete validate :verify_supported - # Keep this scope in sync with the logic in `#resolvable?` - scope :resolvable, -> { user.where(noteable_type: 'MergeRequest') } - scope :resolved, -> { resolvable.where.not(resolved_at: nil) } - scope :unresolved, -> { resolvable.where(resolved_at: nil) } - - after_initialize :ensure_original_discussion_id before_validation :set_original_position, :update_position, on: :create - before_validation :set_line_code, :set_original_discussion_id - # We need to do this again, because it's already in `Note`, but is affected by - # `update_position` and needs to run after that. - before_validation :set_discussion_id + before_validation :set_line_code after_save :keep_around_commits - class << self - def build_discussion_id(noteable_type, noteable_id, position) - [super(noteable_type, noteable_id), *position.key].join("-") - end - - # This method must be kept in sync with `#resolve!` - def resolve!(current_user) - unresolved.update_all(resolved_at: Time.now, resolved_by_id: current_user.id) - end - - # This method must be kept in sync with `#unresolve!` - def unresolve! - resolved.update_all(resolved_at: nil, resolved_by_id: nil) - end + def discussion_class(*) + DiffDiscussion end - def new_diff_note? - true - end + %i(original_position position).each do |meth| + define_method "#{meth}=" do |new_position| + if new_position.is_a?(String) + new_position = JSON.parse(new_position) rescue nil + end - def diff_attributes - { position: position.to_json } - end + if new_position.is_a?(Hash) + new_position = new_position.with_indifferent_access + new_position = Gitlab::Diff::Position.new(new_position) + end - def position=(new_position) - if new_position.is_a?(String) - new_position = JSON.parse(new_position) rescue nil + super(new_position) end - - if new_position.is_a?(Hash) - new_position = new_position.with_indifferent_access - new_position = Gitlab::Diff::Position.new(new_position) - end - - super(new_position) end def diff_file @@ -88,41 +65,11 @@ class DiffNote < Note self.position.diff_refs == diff_refs end - # If you update this method remember to also update the scope `resolvable` - def resolvable? - !system? && for_merge_request? - end - - def resolved? - return false unless resolvable? - - self.resolved_at.present? - end - - # If you update this method remember to also update `.resolve!` - def resolve!(current_user) - return unless resolvable? - return if resolved? - - self.resolved_at = Time.now - self.resolved_by = current_user - save! - end - - # If you update this method remember to also update `.unresolve!` - def unresolve! - return unless resolvable? - return unless resolved? - - self.resolved_at = nil - self.resolved_by = nil - save! - end - - def discussion - return unless resolvable? + def created_at_diff?(diff_refs) + return false unless supported? + return true if for_commit? - self.noteable.find_diff_discussion(self.discussion_id) + self.original_position.diff_refs == diff_refs end private @@ -131,42 +78,14 @@ class DiffNote < Note for_commit? || self.noteable.has_complete_diff_refs? end - def noteable_diff_refs - if noteable.respond_to?(:diff_sha_refs) - noteable.diff_sha_refs - else - noteable.diff_refs - end - end - def set_original_position - self.original_position = self.position.dup + self.original_position = self.position.dup unless self.original_position&.complete? end def set_line_code self.line_code = self.position.line_code(self.project.repository) end - def ensure_original_discussion_id - return unless self.persisted? - return if self.original_discussion_id - - set_original_discussion_id - update_column(:original_discussion_id, self.original_discussion_id) - end - - def set_original_discussion_id - self.original_discussion_id = Digest::SHA1.hexdigest(build_original_discussion_id) - end - - def build_discussion_id - self.class.build_discussion_id(noteable_type, noteable_id || commit_id, position) - end - - def build_original_discussion_id - self.class.build_discussion_id(noteable_type, noteable_id || commit_id, original_position) - end - def update_position return unless supported? return if for_commit? diff --git a/app/models/discussion.rb b/app/models/discussion.rb index bbe813db823..0b6b920ed66 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -1,7 +1,10 @@ +# A non-diff discussion on an issue, merge request, commit, or snippet, consisting of `DiscussionNote` notes. +# +# A discussion of this type can be resolvable. class Discussion - NUMBER_OF_TRUNCATED_DIFF_LINES = 16 + include ResolvableDiscussion - attr_reader :notes + attr_reader :notes, :context_noteable delegate :created_at, :project, @@ -11,43 +14,62 @@ class Discussion :for_commit?, :for_merge_request?, - :line_code, - :original_line_code, - :diff_file, - :for_line?, - :active?, - to: :first_note - delegate :resolved_at, - :resolved_by, + def self.build(notes, context_noteable = nil) + notes.first.discussion_class(context_noteable).new(notes, context_noteable) + end - to: :last_resolved_note, - allow_nil: true + def self.build_collection(notes, context_noteable = nil) + notes.group_by { |n| n.discussion_id(context_noteable) }.values.map { |notes| build(notes, context_noteable) } + end - delegate :blob, - :highlighted_diff_lines, - :diff_lines, + # Returns an alphanumeric discussion ID based on `build_discussion_id` + def self.discussion_id(note) + Digest::SHA1.hexdigest(build_discussion_id(note).join("-")) + end - to: :diff_file, - allow_nil: true + # Returns an array of discussion ID components + def self.build_discussion_id(note) + [*base_discussion_id(note), SecureRandom.hex] + end - def self.for_notes(notes) - notes.group_by(&:discussion_id).values.map { |notes| new(notes) } + def self.base_discussion_id(note) + noteable_id = note.noteable_id || note.commit_id + [:discussion, note.noteable_type.try(:underscore), noteable_id] end - def self.for_diff_notes(notes) - notes.group_by(&:line_code).values.map { |notes| new(notes) } + # When notes on a commit are displayed in context of a merge request that contains that commit, + # these notes are to be displayed as if they were part of one discussion, even though they were actually + # individual notes on the commit with different discussion IDs, so that it's clear that these are not + # notes on the merge request itself. + # + # To turn a list of notes into a list of discussions, they are grouped by discussion ID, so to + # get these out-of-context notes to end up in the same discussion, we need to get them to return the same + # `discussion_id` when this grouping happens. To enable this, `Note#discussion_id` calls out + # to the `override_discussion_id` method on the appropriate `Discussion` subclass, as determined by + # the `discussion_class` method on `Note` or a subclass of `Note`. + # + # If no override is necessary, return `nil`. + # For the case described above, see `OutOfContextDiscussion.override_discussion_id`. + def self.override_discussion_id(note) + nil end - def initialize(notes) - @notes = notes + def self.note_class + DiscussionNote end - def last_resolved_note - return unless resolved? + def initialize(notes, context_noteable = nil) + @notes = notes + @context_noteable = context_noteable + end - @last_resolved_note ||= resolved_notes.sort_by(&:resolved_at).last + def ==(other) + other.class == self.class && + other.context_noteable == self.context_noteable && + other.id == self.id && + other.notes == self.notes end def last_updated_at @@ -59,91 +81,29 @@ class Discussion end def id - first_note.discussion_id + first_note.discussion_id(context_noteable) end alias_method :to_param, :id def diff_discussion? - first_note.diff_note? - end - - def legacy_diff_discussion? - notes.any?(&:legacy_diff_note?) + false end - def resolvable? - return @resolvable if @resolvable.present? - - @resolvable = diff_discussion? && notes.any?(&:resolvable?) + def individual_note? + false end - def resolved? - return @resolved if @resolved.present? - - @resolved = resolvable? && notes.none?(&:to_be_resolved?) - end - - def first_note - @first_note ||= @notes.first - end - - def first_note_to_resolve - @first_note_to_resolve ||= notes.detect(&:to_be_resolved?) + def new_discussion? + notes.length == 1 end def last_note - @last_note ||= @notes.last - end - - def resolved_notes - notes.select(&:resolved?) - end - - def to_be_resolved? - resolvable? && !resolved? - end - - def can_resolve?(current_user) - return false unless current_user - return false unless resolvable? - - current_user == self.noteable.author || - current_user.can?(:resolve_note, self.project) - end - - def resolve!(current_user) - return unless resolvable? - - update { |notes| notes.resolve!(current_user) } - end - - def unresolve! - return unless resolvable? - - update { |notes| notes.unresolve! } - end - - def for_target?(target) - self.noteable == target && !diff_discussion? - end - - def active? - return @active if @active.present? - - @active = first_note.active? + @last_note ||= notes.last end def collapsed? - return false unless diff_discussion? - - if resolvable? - # New diff discussions only disappear once they are marked resolved - resolved? - else - # Old diff discussions disappear once they become outdated - !active? - end + resolved? end def expanded? @@ -151,52 +111,6 @@ class Discussion end def reply_attributes - data = { - noteable_type: first_note.noteable_type, - noteable_id: first_note.noteable_id, - commit_id: first_note.commit_id, - discussion_id: self.id, - } - - if diff_discussion? - data[:note_type] = first_note.type - - data.merge!(first_note.diff_attributes) - end - - data - end - - # Returns an array of at most 16 highlighted lines above a diff note - def truncated_diff_lines(highlight: true) - lines = highlight ? highlighted_diff_lines : diff_lines - prev_lines = [] - - lines.each do |line| - if line.meta? - prev_lines.clear - else - prev_lines << line - - break if for_line?(line) - - prev_lines.shift if prev_lines.length >= NUMBER_OF_TRUNCATED_DIFF_LINES - end - end - - prev_lines - end - - private - - def update - notes_relation = DiffNote.where(id: notes.map(&:id)).fresh - yield(notes_relation) - - # Set the notes array to the updated notes - @notes = notes_relation.to_a - - # Reset the memoized values - @last_resolved_note = @resolvable = @resolved = @first_note = @last_note = nil + first_note.slice(:type, :noteable_type, :noteable_id, :commit_id, :discussion_id) end end diff --git a/app/models/discussion_note.rb b/app/models/discussion_note.rb new file mode 100644 index 00000000000..e660b024083 --- /dev/null +++ b/app/models/discussion_note.rb @@ -0,0 +1,13 @@ +# A note in a non-diff discussion on an issue, merge request, commit, or snippet. +# +# A note of this type can be resolvable. +class DiscussionNote < Note + # Names of all implementers of `Noteable` that support discussions. + NOTEABLE_TYPES = %w(MergeRequest Issue Commit Snippet).freeze + + validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } + + def discussion_class(*) + Discussion + end +end diff --git a/app/models/environment.rb b/app/models/environment.rb index bf33010fd21..61572d8d69a 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -62,7 +62,7 @@ class Environment < ActiveRecord::Base def predefined_variables [ { key: 'CI_ENVIRONMENT_NAME', value: name, public: true }, - { key: 'CI_ENVIRONMENT_SLUG', value: slug, public: true }, + { key: 'CI_ENVIRONMENT_SLUG', value: slug, public: true } ] end @@ -150,7 +150,7 @@ class Environment < ActiveRecord::Base end def metrics - project.monitoring_service.metrics(self) if has_metrics? + project.monitoring_service.environment_metrics(self) if has_metrics? end # An environment name is not necessarily suitable for use in URLs, DNS diff --git a/app/models/event.rb b/app/models/event.rb index 5c34844b5d3..e6fad46077a 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -16,7 +16,7 @@ class Event < ActiveRecord::Base RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour - delegate :name, :email, :public_email, to: :author, prefix: true, allow_nil: true + delegate :name, :email, :public_email, :username, to: :author, prefix: true, allow_nil: true delegate :title, to: :issue, prefix: true, allow_nil: true delegate :title, to: :merge_request, prefix: true, allow_nil: true delegate :title, to: :note, prefix: true, allow_nil: true @@ -30,6 +30,7 @@ class Event < ActiveRecord::Base # Callbacks after_create :reset_project_activity + after_create :set_last_repository_updated_at, if: :push? # Scopes scope :recent, -> { reorder(id: :desc) } @@ -357,4 +358,9 @@ class Event < ActiveRecord::Base def recent_update? project.last_activity_at > RESET_PROJECT_ACTIVITY_INTERVAL.ago end + + def set_last_repository_updated_at + Project.unscoped.where(id: project_id). + update_all(last_repository_updated_at: created_at) + end end diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index 0afbca2cb32..538615130a7 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -36,7 +36,7 @@ class GlobalMilestone closed = count_by_state(milestones_by_state_and_title, 'closed') all = milestones_by_state_and_title.map { |(_, title), _| title }.uniq.count - { + { opened: opened, closed: closed, all: all @@ -86,7 +86,7 @@ class GlobalMilestone end def issues - @issues ||= Issue.of_milestones(milestoneish_ids).includes(:project, :assignee, :labels) + @issues ||= Issue.of_milestones(milestoneish_ids).includes(:project, :assignees, :labels) end def merge_requests @@ -94,7 +94,7 @@ class GlobalMilestone end def participants - @participants ||= milestones.includes(:participants).map(&:participants).flatten.compact.uniq + @participants ||= milestones.map(&:participants).flatten.uniq end def labels diff --git a/app/models/group.rb b/app/models/group.rb index 60274386103..6aab477f431 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -4,6 +4,7 @@ class Group < Namespace include Gitlab::ConfigHelper include Gitlab::VisibilityLevel include AccessRequestable + include Avatarable include Referable include SelectForProjectAuthorization @@ -27,11 +28,14 @@ class Group < Namespace validates :avatar, file_size: { maximum: 200.kilobytes.to_i } + validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } + mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy after_create :post_create_hook after_destroy :post_destroy_hook + after_save :update_two_factor_requirement class << self # Searches for groups matching the given query. @@ -108,10 +112,10 @@ class Group < Namespace allowed_by_projects end - def avatar_url(size = nil) - if self[:avatar].present? - [gitlab_config.url, avatar.url].join - end + def avatar_url(**args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) end def lfs_enabled? @@ -122,7 +126,7 @@ class Group < Namespace end def add_users(users, access_level, current_user: nil, expires_at: nil) - GroupMember.add_users_to_group( + GroupMember.add_users( self, users, access_level, @@ -223,4 +227,12 @@ class Group < Namespace type: public? ? 'O' : 'I' # Open vs Invite-only } end + + protected + + def update_two_factor_requirement + return unless require_two_factor_authentication_changed? || two_factor_grace_period_changed? + + users.find_each(&:update_two_factor_requirement) + end end diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index c631e7a7df5..ee6165fd32d 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -5,7 +5,7 @@ class ProjectHook < WebHook scope :confidential_issue_hooks, -> { where(confidential_issues_events: true) } scope :note_hooks, -> { where(note_events: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true) } - scope :build_hooks, -> { where(build_events: true) } + scope :job_hooks, -> { where(job_events: true) } scope :pipeline_hooks, -> { where(pipeline_events: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true) } end diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 777bad1e724..c645805c6da 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -1,4 +1,9 @@ class SystemHook < WebHook + scope :repository_update_hooks, -> { where(repository_update_events: true) } + + default_value_for :push_events, false + default_value_for :repository_update_events, true + def async_execute(data, hook_name) Sidekiq::Client.enqueue(SystemHookWorker, id, data, hook_name) end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 595602e80fe..a165fdc312f 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -8,8 +8,9 @@ class WebHook < ActiveRecord::Base default_value_for :note_events, false default_value_for :merge_requests_events, false default_value_for :tag_push_events, false - default_value_for :build_events, false + default_value_for :job_events, false default_value_for :pipeline_events, false + default_value_for :repository_update_events, false default_value_for :enable_ssl_verification, true scope :push_hooks, -> { where(push_events: true) } @@ -31,7 +32,7 @@ class WebHook < ActiveRecord::Base post_url = url.gsub("#{parsed_url.userinfo}@", '') auth = { username: CGI.unescape(parsed_url.user), - password: CGI.unescape(parsed_url.password), + password: CGI.unescape(parsed_url.password) } response = WebHook.post(post_url, body: data.to_json, diff --git a/app/models/identity.rb b/app/models/identity.rb index 3bacc450e6e..920a25932b4 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -7,6 +7,8 @@ class Identity < ActiveRecord::Base validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider } validates :user_id, uniqueness: { scope: :provider } + scope :with_extern_uid, ->(provider, extern_uid) { where(extern_uid: extern_uid, provider: provider) } + def ldap? provider.starts_with?('ldap') end diff --git a/app/models/individual_note_discussion.rb b/app/models/individual_note_discussion.rb new file mode 100644 index 00000000000..6be8ca45739 --- /dev/null +++ b/app/models/individual_note_discussion.rb @@ -0,0 +1,17 @@ +# A discussion to wrap a single `Note` note on the root of an issue, merge request, +# commit, or snippet, that is not displayed as a discussion. +# +# A discussion of this type is never resolvable. +class IndividualNoteDiscussion < Discussion + def self.note_class + Note + end + + def individual_note? + true + end + + def reply_attributes + super.tap { |attrs| attrs.delete(:discussion_id) } + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 10a5d9d2a24..a88dbb3e065 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -3,6 +3,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include InternalId include Issuable + include Noteable include Referable include Sortable include Spammable @@ -23,12 +24,17 @@ class Issue < ActiveRecord::Base has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + has_many :issue_assignees + has_many :assignees, class_name: "User", through: :issue_assignees + validates :project, presence: true - scope :cared, ->(user) { where(assignee_id: user) } - scope :open_for, ->(user) { opened.assigned_to(user) } scope :in_projects, ->(project_ids) { where(project_id: project_ids) } + scope :assigned, -> { where('EXISTS (SELECT TRUE FROM issue_assignees WHERE issue_id = issues.id)') } + scope :unassigned, -> { where('NOT EXISTS (SELECT TRUE FROM issue_assignees WHERE issue_id = issues.id)') } + scope :assigned_to, ->(u) { where('EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = ? AND issue_id = issues.id)', u.id)} + scope :without_due_date, -> { where(due_date: nil) } scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } @@ -38,11 +44,15 @@ class Issue < ActiveRecord::Base scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } - scope :include_associations, -> { includes(:assignee, :labels, project: :namespace) } + scope :include_associations, -> { includes(:labels, project: :namespace) } + + after_save :expire_etag_cache attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true + participant :assignees + state_machine :state, initial: :opened do event :close do transition [:reopened, :opened] => :closed @@ -59,17 +69,17 @@ class Issue < ActiveRecord::Base before_transition any => :closed do |issue| issue.closed_at = Time.zone.now end - - before_transition closed: any do |issue| - issue.closed_at = nil - end end def hook_attrs + assignee_ids = self.assignee_ids + attrs = { total_time_spent: total_time_spent, human_total_time_spent: human_total_time_spent, - human_time_estimate: human_time_estimate + human_time_estimate: human_time_estimate, + assignee_ids: assignee_ids, + assignee_id: assignee_ids.first # This key is deprecated } attributes.merge!(attrs) @@ -117,6 +127,22 @@ class Issue < ActiveRecord::Base "id DESC") end + # Returns a Hash of attributes to be used for Twitter card metadata + def card_attributes + { + 'Author' => author.try(:name), + 'Assignee' => assignee_list + } + end + + def assignee_or_author?(user) + author_id == user.id || assignees.exists?(user.id) + end + + def assignee_list + assignees.map(&:name).to_sentence + end + # `from` argument can be a Namespace or Project. def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" @@ -146,6 +172,14 @@ class Issue < ActiveRecord::Base branches_with_iid - branches_with_merge_request end + # Returns boolean if a related branch exists for the current issue + # ignores merge requests branchs + def has_related_branch? + project.repository.branch_names.any? do |branch| + /\A#{iid}-(?!\d+-stable)/i =~ branch + end + end + # To allow polymorphism with MergeRequest. def source_project project @@ -202,7 +236,7 @@ class Issue < ActiveRecord::Base # Returns `true` if the current issue can be viewed by either a logged in User # or an anonymous user. def visible_to_user?(user = nil) - return false unless project.feature_available?(:issues, user) + return false unless project && project.feature_available?(:issues, user) user ? readable_by?(user) : publicly_visible? end @@ -243,7 +277,7 @@ class Issue < ActiveRecord::Base true elsif confidential? author == user || - assignee == user || + assignees.include?(user) || project.team.member?(user, Gitlab::Access::REPORTER) else project.public? || @@ -256,4 +290,13 @@ class Issue < ActiveRecord::Base def publicly_visible? project.public? && !confidential? end + + def expire_etag_cache + key = Gitlab::Routing.url_helpers.realtime_changes_namespace_project_issue_path( + project.namespace, + project, + self + ) + Gitlab::EtagCaching::Store.new.touch(key) + end end diff --git a/app/models/issue_assignee.rb b/app/models/issue_assignee.rb new file mode 100644 index 00000000000..06d760b6a89 --- /dev/null +++ b/app/models/issue_assignee.rb @@ -0,0 +1,6 @@ +class IssueAssignee < ActiveRecord::Base + extend Gitlab::CurrentSettings + + belongs_to :issue + belongs_to :assignee, class_name: "User", foreign_key: :user_id +end diff --git a/app/models/key.rb b/app/models/key.rb index 9c74ca84753..b7956052c3f 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -74,7 +74,7 @@ class Key < ActiveRecord::Base GitlabShellWorker.perform_async( :remove_key, shell_id, - key, + key ) end diff --git a/app/models/label.rb b/app/models/label.rb index 568fa6d44f5..ddddb6bdf8f 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -21,6 +21,8 @@ class Label < ActiveRecord::Base has_many :issues, through: :label_links, source: :target, source_type: 'Issue' has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' + before_validation :strip_whitespace_from_title_and_color + validates :color, color: true, allow_blank: false # Don't allow ',' for label titles @@ -32,6 +34,7 @@ class Label < ActiveRecord::Base scope :templates, -> { where(template: true) } scope :with_title, ->(title) { where(title: title) } + scope :on_project_boards, ->(project_id) { joins(lists: :board).merge(List.movable).where(boards: { project_id: project_id }) } def self.prioritized(project) joins(:priorities) @@ -193,4 +196,8 @@ class Label < ActiveRecord::Base def sanitize_title(value) CGI.unescapeHTML(Sanitize.clean(value.to_s)) end + + def strip_whitespace_from_title_and_color + %w(color title).each { |attr| self[attr] = self[attr]&.strip } + end end diff --git a/app/models/legacy_diff_discussion.rb b/app/models/legacy_diff_discussion.rb new file mode 100644 index 00000000000..3c1d34db5fa --- /dev/null +++ b/app/models/legacy_diff_discussion.rb @@ -0,0 +1,43 @@ +# A discussion on merge request or commit diffs consisting of `LegacyDiffNote` notes. +# +# All new diff discussions are of the type `DiffDiscussion`, but any diff discussions created +# before the introduction of the new implementation still use `LegacyDiffDiscussion`. +# +# A discussion of this type is never resolvable. +class LegacyDiffDiscussion < Discussion + include DiscussionOnDiff + + memoized_values << :active + + def self.note_class + LegacyDiffNote + end + + def legacy_diff_discussion? + true + end + + def active?(*args) + return @active if @active.present? + + @active = first_note.active?(*args) + end + + def collapsed? + !active? + end + + def merge_request_version_params + return unless for_merge_request? + + if active? + {} + else + nil + end + end + + def reply_attributes + super.merge(line_code: line_code) + end +end diff --git a/app/models/legacy_diff_note.rb b/app/models/legacy_diff_note.rb index 40277a9b139..d7c627432d2 100644 --- a/app/models/legacy_diff_note.rb +++ b/app/models/legacy_diff_note.rb @@ -1,3 +1,9 @@ +# A note on merge request or commit diffs, using the legacy implementation. +# +# All new diff notes are of the type `DiffNote`, but any diff notes created +# before the introduction of the new implementation still use `LegacyDiffNote`. +# +# A note of this type is never resolvable. class LegacyDiffNote < Note include NoteOnDiff @@ -7,18 +13,8 @@ class LegacyDiffNote < Note before_create :set_diff - class << self - def build_discussion_id(noteable_type, noteable_id, line_code) - [super(noteable_type, noteable_id), line_code].join("-") - end - end - - def legacy_diff_note? - true - end - - def diff_attributes - { line_code: line_code } + def discussion_class(*) + LegacyDiffDiscussion end def project_repository @@ -60,11 +56,12 @@ class LegacyDiffNote < Note # # If the note's current diff cannot be matched in the MergeRequest's current # diff, it's considered inactive. - def active? + def active?(diff_refs = nil) return @active if defined?(@active) return true if for_commit? return true unless diff_line return false unless noteable + return false if diff_refs && diff_refs != noteable_diff_refs noteable_diff = find_noteable_diff @@ -119,8 +116,4 @@ class LegacyDiffNote < Note diffs = noteable.raw_diffs(Commit.max_diff_options) diffs.find { |d| d.new_path == self.diff.new_path } end - - def build_discussion_id - self.class.build_discussion_id(noteable_type, noteable_id || commit_id, line_code) - end end diff --git a/app/models/member.rb b/app/models/member.rb index 0545bd4eedf..7228e82e978 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -151,6 +151,27 @@ class Member < ActiveRecord::Base member end + def add_users(source, users, access_level, current_user: nil, expires_at: nil) + return [] unless users.present? + + # Collect all user ids into separate array + # so we can use single sql query to get user objects + user_ids = users.select { |user| user =~ /\A\d+\Z/ } + users = users - user_ids + User.where(id: user_ids) + + self.transaction do + users.map do |user| + add_user( + source, + user, + access_level, + current_user: current_user, + expires_at: expires_at + ) + end + end + end + def access_levels Gitlab::Access.sym_options end @@ -173,18 +194,6 @@ class Member < ActiveRecord::Base # There is no current user for bulk actions, in which case anything is allowed !current_user || current_user.can?(:"update_#{member.type.underscore}", member) end - - def add_users_to_source(source, users, access_level, current_user: nil, expires_at: nil) - users.each do |user| - add_user( - source, - user, - access_level, - current_user: current_user, - expires_at: expires_at - ) - end - end end def real_source_type diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 446f9f8f8a7..28e10bc6172 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -3,11 +3,16 @@ class GroupMember < Member belongs_to :group, foreign_key: 'source_id' + delegate :update_two_factor_requirement, to: :user + # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE validates :source_type, format: { with: /\ANamespace\z/ } default_scope { where(source_type: SOURCE_TYPE) } + after_create :update_two_factor_requirement, unless: :invite? + after_destroy :update_two_factor_requirement, unless: :invite? + def self.access_level_roles Gitlab::Access.options_with_owner end @@ -16,18 +21,6 @@ class GroupMember < Member Gitlab::Access.sym_options_with_owner end - def self.add_users_to_group(group, users, access_level, current_user: nil, expires_at: nil) - self.transaction do - add_users_to_source( - group, - users, - access_level, - current_user: current_user, - expires_at: expires_at - ) - end - end - def group source end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 912820b51ac..b3a91feb091 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -16,7 +16,7 @@ class ProjectMember < Member before_destroy :delete_member_todos class << self - # Add users to project teams with passed access option + # Add users to projects with passed access option # # access can be an integer representing a access code # or symbol like :master representing role @@ -39,7 +39,7 @@ class ProjectMember < Member project_ids.each do |project_id| project = Project.find(project_id) - add_users_to_source( + add_users( project, users, access_level, diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 5ff83944d8c..9be00880438 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,9 +1,9 @@ class MergeRequest < ActiveRecord::Base include InternalId include Issuable + include Noteable include Referable include Sortable - include Importable belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" @@ -13,10 +13,14 @@ class MergeRequest < ActiveRecord::Base has_one :merge_request_diff, -> { order('merge_request_diffs.id DESC') } + belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" + has_many :events, as: :target, dependent: :destroy has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + belongs_to :assignee, class_name: "User" + serialize :merge_params, Hash after_create :ensure_merge_request_diff, unless: :importing? @@ -100,11 +104,11 @@ class MergeRequest < ActiveRecord::Base validates :merge_user, presence: true, if: :merge_when_pipeline_succeeds?, unless: :importing? validate :validate_branches, unless: [:allow_broken, :importing?, :closed_without_fork?] validate :validate_fork, unless: :closed_without_fork? + validate :validate_target_project, on: :create scope :by_source_or_target_branch, ->(branch_name) do where("source_branch = :branch OR target_branch = :branch", branch: branch_name) end - scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) } scope :by_milestone, ->(milestone) { where(milestone_id: milestone) } scope :of_projects, ->(ids) { where(target_project_id: ids) } scope :from_project, ->(project) { where(source_project_id: project.id) } @@ -114,6 +118,11 @@ class MergeRequest < ActiveRecord::Base scope :join_project, -> { joins(:target_project) } scope :references_project, -> { references(:target_project) } + scope :assigned, -> { where("assignee_id IS NOT NULL") } + scope :unassigned, -> { where("assignee_id IS NULL") } + scope :assigned_to, ->(u) { where(assignee_id: u.id)} + + participant :assignee after_save :keep_around_commit @@ -177,6 +186,23 @@ class MergeRequest < ActiveRecord::Base work_in_progress?(title) ? title : "WIP: #{title}" end + # Returns a Hash of attributes to be used for Twitter card metadata + def card_attributes + { + 'Author' => author.try(:name), + 'Assignee' => assignee.try(:name) + } + end + + # This method is needed for compatibility with issues to not mess view and other code + def assignees + Array(assignee) + end + + def assignee_or_author?(user) + author_id == user.id || assignee_id == user.id + end + # `from` argument can be a Namespace or Project. def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" @@ -192,22 +218,23 @@ class MergeRequest < ActiveRecord::Base merge_request_diff ? merge_request_diff.raw_diffs(*args) : compare.raw_diffs(*args) end - def diffs(diff_options = nil) + def diffs(diff_options = {}) if compare - compare.diffs(diff_options) + # When saving MR diffs, `no_collapse` is implicitly added (because we need + # to save the entire contents to the DB), so add that here for + # consistency. + compare.diffs(diff_options.merge(no_collapse: true)) else merge_request_diff.diffs(diff_options) end end def diff_size - # The `#diffs` method ends up at an instance of a class inheriting from - # `Gitlab::Diff::FileCollection::Base`, so use those options as defaults - # here too, to get the same diff size without performing highlighting. - # - opts = Gitlab::Diff::FileCollection::Base.default_options.merge(diff_options || {}) + # Calling `merge_request_diff.diffs.real_size` will also perform + # highlighting, which we don't need here. + return real_size if merge_request_diff - raw_diffs(opts).size + diffs.real_size end def diff_base_commit @@ -266,6 +293,8 @@ class MergeRequest < ActiveRecord::Base attr_writer :target_branch_sha, :source_branch_sha def source_branch_head + return unless source_project + source_branch_ref = @source_branch_sha || source_branch source_project.repository.commit(source_branch_ref) if source_branch_ref end @@ -330,6 +359,12 @@ class MergeRequest < ActiveRecord::Base end end + def validate_target_project + return true if target_project.merge_requests_enabled? + + errors.add :base, 'Target project has disabled merge requests' + end + def validate_fork return true unless target_project && source_project return true if target_project == source_project @@ -367,6 +402,20 @@ class MergeRequest < ActiveRecord::Base merge_request_diff(true) end + def merge_request_diff_for(diff_refs_or_sha) + @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha| + diffs = merge_request_diffs.viewable.select_without_diff + h[diff_refs_or_sha] = + if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) + diffs.find_by_diff_refs(diff_refs_or_sha) + else + diffs.find_by(head_commit_sha: diff_refs_or_sha) + end + end + + @merge_request_diffs_by_diff_refs_or_sha[diff_refs_or_sha] + end + def reload_diff_if_branch_changed if source_branch_changed? || target_branch_changed? reload_diff @@ -443,7 +492,7 @@ class MergeRequest < ActiveRecord::Base end def can_remove_source_branch?(current_user) - !source_project.protected_branch?(source_branch) && + !ProtectedBranch.protected?(source_project, source_branch) && !source_project.root_ref?(source_branch) && Ability.allowed?(current_user, :push_code, source_project) && diff_head_commit == source_branch_head @@ -476,43 +525,7 @@ class MergeRequest < ActiveRecord::Base ) end - def discussions - @discussions ||= self.related_notes. - inc_relations_for_view. - fresh. - discussions - end - - def diff_discussions - @diff_discussions ||= self.notes.diff_notes.discussions - end - - def resolvable_discussions - @resolvable_discussions ||= diff_discussions.select(&:to_be_resolved?) - end - - def discussions_can_be_resolved_by?(user) - resolvable_discussions.all? { |discussion| discussion.can_resolve?(user) } - end - - def find_diff_discussion(discussion_id) - notes = self.notes.diff_notes.where(discussion_id: discussion_id).fresh.to_a - return if notes.empty? - - Discussion.new(notes) - end - - def discussions_resolvable? - diff_discussions.any?(&:resolvable?) - end - - def discussions_resolved? - discussions_resolvable? && diff_discussions.none?(&:to_be_resolved?) - end - - def discussions_to_be_resolved? - discussions_resolvable? && !discussions_resolved? - end + alias_method :discussion_notes, :related_notes def mergeable_discussions_state? return true unless project.only_allow_merge_if_all_discussions_are_resolved? @@ -812,12 +825,6 @@ class MergeRequest < ActiveRecord::Base diverged_commits_count > 0 end - def head_pipeline - return unless diff_head_sha && source_project - - @head_pipeline ||= source_project.pipeline_for(source_branch, diff_head_sha) - end - def all_pipelines return Ci::Pipeline.none unless source_project @@ -847,7 +854,7 @@ class MergeRequest < ActiveRecord::Base end def can_be_cherry_picked? - merge_commit + merge_commit.present? end def has_complete_diff_refs? @@ -858,8 +865,8 @@ class MergeRequest < ActiveRecord::Base return unless has_complete_diff_refs? return if new_diff_refs == old_diff_refs - active_diff_notes = self.notes.diff_notes.select do |note| - note.new_diff_note? && note.active?(old_diff_refs) + active_diff_notes = self.notes.new_diff_notes.select do |note| + note.active?(old_diff_refs) end return if active_diff_notes.empty? @@ -886,32 +893,6 @@ class MergeRequest < ActiveRecord::Base project.repository.keep_around(self.merge_commit_sha) end - def conflicts - @conflicts ||= Gitlab::Conflict::FileCollection.new(self) - end - - def conflicts_can_be_resolved_by?(user) - access = ::Gitlab::UserAccess.new(user, project: source_project) - access.can_push_to_branch?(source_branch) - end - - def conflicts_can_be_resolved_in_ui? - return @conflicts_can_be_resolved_in_ui if defined?(@conflicts_can_be_resolved_in_ui) - - return @conflicts_can_be_resolved_in_ui = false unless cannot_be_merged? - return @conflicts_can_be_resolved_in_ui = false unless has_complete_diff_refs? - - begin - # Try to parse each conflict. If the MR's mergeable status hasn't been updated, - # ensure that we don't say there are conflicts to resolve when there are no conflict - # files. - conflicts.files.each(&:lines) - @conflicts_can_be_resolved_in_ui = conflicts.files.length > 0 - rescue Rugged::OdbError, Gitlab::Conflict::Parser::UnresolvableError, Gitlab::Conflict::FileCollection::ConflictSideMissing - @conflicts_can_be_resolved_in_ui = false - end - end - def has_commits? merge_request_diff && commits_count > 0 end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index baee00b8fcd..f0a3c30ea74 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -31,6 +31,10 @@ class MergeRequestDiff < ActiveRecord::Base # It allows you to override variables like head_commit_sha before getting diff. after_create :save_git_content, unless: :importing? + def self.find_by_diff_refs(diff_refs) + find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) + end + def self.select_without_diff select(column_names - ['st_diffs']) end @@ -130,6 +134,12 @@ class MergeRequestDiff < ActiveRecord::Base st_commits.map { |commit| commit[:id] } end + def diff_refs=(new_diff_refs) + self.base_commit_sha = new_diff_refs&.base_sha + self.start_commit_sha = new_diff_refs&.start_sha + self.head_commit_sha = new_diff_refs&.head_sha + end + def diff_refs return unless start_commit_sha || base_commit_sha @@ -177,6 +187,16 @@ class MergeRequestDiff < ActiveRecord::Base st_commits.count end + def utf8_st_diffs + return [] if st_diffs.blank? + + st_diffs.map do |diff| + diff.each do |k, v| + diff[k] = encode_utf8(v) if v.respond_to?(:encoding) + end + end + end + private # Old GitLab implementations may have generated diffs as ["--broken-diff"]. @@ -240,7 +260,7 @@ class MergeRequestDiff < ActiveRecord::Base new_attributes[:state] = :empty else diff_collection = compare.diffs(Commit.max_diff_options) - new_attributes[:real_size] = compare.diffs.real_size + new_attributes[:real_size] = diff_collection.real_size if diff_collection.any? new_diffs = dump_diffs(diff_collection) @@ -270,14 +290,6 @@ class MergeRequestDiff < ActiveRecord::Base project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha) end - def utf8_st_diffs - st_diffs.map do |diff| - diff.each do |k, v| - diff[k] = encode_utf8(v) if v.respond_to?(:encoding) - end - end - end - # # #save or #update_attributes providing changes on serialized attributes do a lot of # serialization and deserialization calls resulting in bad performance. diff --git a/app/models/milestone.rb b/app/models/milestone.rb index e85d5709624..c06bfe0ccdd 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -21,7 +21,6 @@ class Milestone < ActiveRecord::Base has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests - has_many :participants, -> { distinct.reorder('users.name') }, through: :issues, source: :assignee has_many :events, as: :target, dependent: :destroy scope :active, -> { with_state(:active) } @@ -30,7 +29,7 @@ class Milestone < ActiveRecord::Base validates :title, presence: true, uniqueness: { scope: :project_id } validates :project, presence: true - validate :start_date_should_be_less_than_due_date, if: Proc.new { |m| m.start_date.present? && m.due_date.present? } + validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } strip_attributes :title @@ -107,6 +106,10 @@ class Milestone < ActiveRecord::Base end end + def participants + User.joins(assigned_issues: :milestone).where("milestones.id = ?", id) + end + def self.sort(method) case method.to_s when 'due_date_asc' @@ -153,10 +156,6 @@ class Milestone < ActiveRecord::Base active? && issues.opened.count.zero? end - def is_empty?(user = nil) - total_items_count(user).zero? - end - def author_id nil end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 1d4b1f7d590..4d59267f71d 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -33,7 +33,7 @@ class Namespace < ActiveRecord::Base validates :path, presence: true, length: { maximum: 255 }, - namespace: true + dynamic_path: true validate :nesting_level_allowed @@ -46,7 +46,7 @@ class Namespace < ActiveRecord::Base before_destroy(prepend: true) { prepare_for_destroy } after_destroy :rm_dir - scope :root, -> { where('type IS NULL') } + scope :for_user, -> { where('type IS NULL') } scope :with_statistics, -> do joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id') @@ -56,7 +56,7 @@ class Namespace < ActiveRecord::Base 'COALESCE(SUM(ps.storage_size), 0) AS storage_size', 'COALESCE(SUM(ps.repository_size), 0) AS repository_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', - 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', + 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size' ) end @@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_tags? - projects.any?(&:has_container_registry_tags?) + all_projects.any?(&:has_container_registry_tags?) end def send_update_instructions @@ -214,6 +214,16 @@ class Namespace < ActiveRecord::Base @old_repository_storage_paths ||= repository_storage_paths end + # Includes projects from this namespace and projects from all subgroups + # that belongs to this namespace + def all_projects + Project.inside_path(full_path) + end + + def has_parent? + parent.present? + end + private def repository_storage_paths @@ -221,7 +231,7 @@ class Namespace < ActiveRecord::Base # pending delete. Unscoping also get rids of the default order, which causes # problems with SELECT DISTINCT. Project.unscoped do - projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) end end diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 0bbc9451ffd..59737bb6085 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -107,7 +107,8 @@ module Network def find_commits(skip = 0) opts = { max_count: self.class.max_count, - skip: skip + skip: skip, + order: :date } opts[:ref] = @commit.id if @filter_ref diff --git a/app/models/note.rb b/app/models/note.rb index 16d66cb1427..46d0a4f159f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -1,3 +1,6 @@ +# A note on the root of an issue, merge request, commit, or snippet. +# +# A note of this type is never resolvable. class Note < ActiveRecord::Base extend ActiveModel::Naming include Gitlab::CurrentSettings @@ -8,8 +11,17 @@ class Note < ActiveRecord::Base include FasterCacheKeys include CacheMarkdownField include AfterCommitQueue + include ResolvableNote + include IgnorableColumn - cache_markdown_field :note, pipeline: :note + ignore_column :original_discussion_id + + cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true + + # Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with notes. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10392/diffs#note_28719102 + alias_attribute :last_edited_at, :updated_at + alias_attribute :last_edited_by, :updated_by # Attribute containing rendered and redacted Markdown as generated by # Banzai::ObjectRenderer. @@ -31,9 +43,7 @@ class Note < ActiveRecord::Base belongs_to :noteable, polymorphic: true, touch: true belongs_to :author, class_name: "User" belongs_to :updated_by, class_name: "User" - - # Only used by DiffNote, but defined here so that it can be used in `Note.includes` - belongs_to :resolved_by, class_name: "User" + belongs_to :last_edited_by, class_name: 'User' has_many :todos, dependent: :destroy has_many :events, as: :target, dependent: :destroy @@ -54,10 +64,11 @@ class Note < ActiveRecord::Base validates :noteable_id, presence: true, unless: [:for_commit?, :importing?] validates :commit_id, presence: true, if: :for_commit? validates :author, presence: true + validates :discussion_id, presence: true, format: { with: /\A\h{40}\z/ } validate unless: [:for_commit?, :importing?, :for_personal_snippet?] do |note| unless note.noteable.try(:project) == note.project - errors.add(:invalid_project, 'Note and noteable project mismatch') + errors.add(:project, 'does not match noteable project') end end @@ -69,6 +80,7 @@ class Note < ActiveRecord::Base scope :user, ->{ where(system: false) } scope :common, ->{ where(noteable_type: ["", nil]) } scope :fresh, ->{ order(created_at: :asc, id: :asc) } + scope :updated_after, ->(time){ where('updated_at > ?', time) } scope :inc_author_project, ->{ includes(:project, :author) } scope :inc_author, ->{ includes(:author) } scope :inc_relations_for_view, -> do @@ -76,7 +88,8 @@ class Note < ActiveRecord::Base end scope :diff_notes, ->{ where(type: %w(LegacyDiffNote DiffNote)) } - scope :non_diff_notes, ->{ where(type: ['Note', nil]) } + scope :new_diff_notes, ->{ where(type: 'DiffNote') } + scope :non_diff_notes, ->{ where(type: ['Note', 'DiscussionNote', nil]) } scope :with_associations, -> do # FYI noteable cannot be loaded for LegacyDiffNote for commits @@ -86,31 +99,41 @@ class Note < ActiveRecord::Base after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code - before_validation :set_discussion_id + before_validation :set_discussion_id, on: :create after_save :keep_around_commit, unless: :for_personal_snippet? after_save :expire_etag_cache + after_destroy :expire_etag_cache class << self def model_name ActiveModel::Name.new(self, nil, 'note') end - def build_discussion_id(noteable_type, noteable_id) - [:discussion, noteable_type.try(:underscore), noteable_id].join("-") + def discussions(context_noteable = nil) + Discussion.build_collection(fresh, context_noteable) end - def discussion_id(*args) - Digest::SHA1.hexdigest(build_discussion_id(*args)) - end + def find_discussion(discussion_id) + notes = where(discussion_id: discussion_id).fresh.to_a + return if notes.empty? - def discussions - Discussion.for_notes(fresh) + Discussion.build(notes) end - def grouped_diff_discussions - active_notes = diff_notes.fresh.select(&:active?) - Discussion.for_diff_notes(active_notes). - map { |d| [d.line_code, d] }.to_h + def grouped_diff_discussions(diff_refs = nil) + groups = {} + + diff_notes.fresh.discussions.each do |discussion| + if discussion.active?(diff_refs) + discussions = groups[discussion.line_code] ||= [] + elsif diff_refs && discussion.created_at_diff?(diff_refs) + discussions = groups[discussion.original_line_code] ||= [] + end + + discussions << discussion if discussions + end + + groups end def count_for_collection(ids, type) @@ -121,37 +144,17 @@ class Note < ActiveRecord::Base end def cross_reference? - system && SystemNoteService.cross_reference?(note) + system? && SystemNoteService.cross_reference?(note) end def diff_note? false end - def legacy_diff_note? - false - end - - def new_diff_note? - false - end - def active? true end - def resolvable? - false - end - - def resolved? - false - end - - def to_be_resolved? - resolvable? && !resolved? - end - def max_attachment_size current_application_settings.max_attachment_size.megabytes.to_i end @@ -228,7 +231,7 @@ class Note < ActiveRecord::Base end def can_be_award_emoji? - noteable.is_a?(Awardable) + noteable.is_a?(Awardable) && !part_of_discussion? end def contains_emoji_only? @@ -239,6 +242,63 @@ class Note < ActiveRecord::Base for_personal_snippet? ? 'personal_snippet' : noteable_type.underscore end + def can_be_discussion_note? + self.noteable.supports_discussions? && !part_of_discussion? + end + + def discussion_class(noteable = nil) + # When commit notes are rendered on an MR's Discussion page, they are + # displayed in one discussion instead of individually. + # See also `#discussion_id` and `Discussion.override_discussion_id`. + if noteable && noteable != self.noteable + OutOfContextDiscussion + else + IndividualNoteDiscussion + end + end + + # See `Discussion.override_discussion_id` for details. + def discussion_id(noteable = nil) + discussion_class(noteable).override_discussion_id(self) || super() + end + + # Returns a discussion containing just this note. + # This method exists as an alternative to `#discussion` to use when the methods + # we intend to call on the Discussion object don't require it to have all of its notes, + # and just depend on the first note or the type of discussion. This saves us a DB query. + def to_discussion(noteable = nil) + Discussion.build([self], noteable) + end + + # Returns the entire discussion this note is part of. + # Consider using `#to_discussion` if we do not need to render the discussion + # and all its notes and if we don't care about the discussion's resolvability status. + def discussion + full_discussion = self.noteable.notes.find_discussion(self.discussion_id) if part_of_discussion? + full_discussion || to_discussion + end + + def part_of_discussion? + !to_discussion.individual_note? + end + + def in_reply_to?(other) + case other + when Note + if part_of_discussion? + in_reply_to?(other.noteable) && in_reply_to?(other.to_discussion) + else + in_reply_to?(other.noteable) + end + when Discussion + self.discussion_id == other.id + when Noteable + self.noteable == other + else + false + end + end + private def keep_around_commit @@ -264,17 +324,7 @@ class Note < ActiveRecord::Base end def set_discussion_id - self.discussion_id = Digest::SHA1.hexdigest(build_discussion_id) - end - - def build_discussion_id - if for_merge_request? - # Notes on merge requests are always in a discussion of their own, - # so we generate a unique discussion ID. - [:discussion, :note, SecureRandom.hex].join("-") - else - self.class.build_discussion_id(noteable_type, noteable_id || commit_id) - end + self.discussion_id ||= discussion_class.discussion_id(self) end def expire_etag_cache diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 52577bd52ea..e4726e62e93 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -60,16 +60,25 @@ class NotificationSetting < ActiveRecord::Base def set_events return if custom? - EMAIL_EVENTS.each do |event| - events[event] = false - end + self.events = {} end # Validates store accessors values as boolean # It is a text field so it does not cast correct boolean values in JSON def events_to_boolean EMAIL_EVENTS.each do |event| - events[event] = ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(events[event]) + bool = ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(public_send(event)) + + events[event] = bool end end + + # Allow people to receive failed pipeline notifications if they already have + # custom notifications enabled, as these are more like mentions than the other + # custom settings. + def failed_pipeline + bool = super + + bool.nil? || bool + end end diff --git a/app/models/out_of_context_discussion.rb b/app/models/out_of_context_discussion.rb new file mode 100644 index 00000000000..4227c40b69a --- /dev/null +++ b/app/models/out_of_context_discussion.rb @@ -0,0 +1,26 @@ +# When notes on a commit are displayed in the context of a merge request that +# contains that commit, they are displayed as if they were a discussion. +# +# This represents one of those discussions, consisting of `Note` notes. +# +# A discussion of this type is never resolvable. +class OutOfContextDiscussion < Discussion + # Returns an array of discussion ID components + def self.build_discussion_id(note) + base_discussion_id(note) + end + + # To make sure all out-of-context notes end up grouped as one discussion, + # we override the discussion ID to be a newly generated but consistent ID. + def self.override_discussion_id(note) + discussion_id(note) + end + + def self.note_class + Note + end + + def reply_attributes + super.tap { |attrs| attrs.delete(:discussion_id) } + end +end diff --git a/app/models/project.rb b/app/models/project.rb index f1bba56d32c..65745fd6d37 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -6,6 +6,7 @@ class Project < ActiveRecord::Base include Gitlab::VisibilityLevel include Gitlab::CurrentSettings include AccessRequestable + include Avatarable include CacheMarkdownField include Referable include Sortable @@ -53,6 +54,11 @@ class Project < ActiveRecord::Base update_column(:last_activity_at, self.created_at) end + after_create :set_last_repository_updated_at + def set_last_repository_updated_at + update_column(:last_repository_updated_at, self.created_at) + end + after_destroy :remove_pages # update visibility_level of forks @@ -74,6 +80,7 @@ class Project < ActiveRecord::Base attr_accessor :new_default_branch attr_accessor :old_path_with_namespace + attr_writer :pipeline_status alias_attribute :title, :name @@ -114,6 +121,9 @@ class Project < ActiveRecord::Base has_one :kubernetes_service, dependent: :destroy, inverse_of: :project has_one :prometheus_service, dependent: :destroy, inverse_of: :project has_one :mock_ci_service, dependent: :destroy + has_one :mock_deployment_service, dependent: :destroy + has_one :mock_monitoring_service, dependent: :destroy + has_one :microsoft_teams_service, dependent: :destroy has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" has_one :forked_from_project, through: :forked_project_link @@ -132,6 +142,7 @@ class Project < ActiveRecord::Base has_many :snippets, dependent: :destroy, class_name: 'ProjectSnippet' has_many :hooks, dependent: :destroy, class_name: 'ProjectHook' has_many :protected_branches, dependent: :destroy + has_many :protected_tags, dependent: :destroy has_many :project_authorizations has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' @@ -157,16 +168,20 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete + has_many :container_repositories, dependent: :destroy has_many :commit_statuses, dependent: :destroy has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline' has_many :builds, class_name: 'Ci::Build' # the builds are created from the commit_statuses has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject' has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' - has_many :variables, dependent: :destroy, class_name: 'Ci::Variable' + has_many :variables, class_name: 'Ci::Variable' has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger' has_many :environments, dependent: :destroy has_many :deployments, dependent: :destroy + has_many :pipeline_schedules, dependent: :destroy, class_name: 'Ci::PipelineSchedule' + + has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature @@ -174,7 +189,7 @@ class Project < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true delegate :count, to: :forks, prefix: true delegate :members, to: :team, prefix: true - delegate :add_user, to: :team + delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team delegate :empty_repo?, to: :repository @@ -188,13 +203,14 @@ class Project < ActiveRecord::Base message: Gitlab::Regex.project_name_regex_message } validates :path, presence: true, - project_path: true, + dynamic_path: true, length: { maximum: 255 }, format: { with: Gitlab::Regex.project_path_regex, - message: Gitlab::Regex.project_path_regex_message } + message: Gitlab::Regex.project_path_regex_message }, + uniqueness: { scope: :namespace_id } + validates :namespace, presence: true validates :name, uniqueness: { scope: :namespace_id } - validates :path, uniqueness: { scope: :namespace_id } validates :import_url, addressable_url: true, if: :external_import? validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } @@ -256,6 +272,8 @@ class Project < ActiveRecord::Base scope :with_builds_enabled, -> { with_feature_enabled(:builds) } scope :with_issues_enabled, -> { with_feature_enabled(:issues) } + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } + # project features may be "disabled", "internal" or "enabled". If "internal", # they are only available to team members. This scope returns projects where # the feature is either enabled, or internal with permission for the user. @@ -347,10 +365,15 @@ class Project < ActiveRecord::Base end def sort(method) - if method == 'storage_size_desc' + case method.to_s + when 'storage_size_desc' # storage_size is a joined column so we need to # pass a string to avoid AR adding the table name reorder('project_statistics.storage_size DESC, projects.id DESC') + when 'latest_activity_desc' + reorder(last_activity_at: :desc) + when 'latest_activity_asc' + reorder(last_activity_at: :asc) else order_by(method) end @@ -399,32 +422,15 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry_path_with_namespace - path_with_namespace.downcase - end - - def container_registry_repository - return unless Gitlab.config.registry.enabled - - @container_registry_repository ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_path_with_namespace) - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - registry = ContainerRegistry::Registry.new(url, token: token, path: host_port) - registry.repository(container_registry_path_with_namespace) - end - end - - def container_registry_repository_url + def container_registry_url if Gitlab.config.registry.enabled - "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}" + "#{Gitlab.config.registry.host_port}/#{path_with_namespace.downcase}" end end def has_container_registry_tags? - return unless container_registry_repository - - container_registry_repository.tags.any? + container_repositories.to_a.any?(&:has_tags?) || + has_root_container_repository_tags? end def commit(ref = 'HEAD') @@ -551,6 +557,10 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def github_import? + import_type == 'github' + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit @@ -789,12 +799,10 @@ class Project < ActiveRecord::Base repository.avatar end - def avatar_url - if self[:avatar].present? - [gitlab_config.url, avatar.url].join - elsif avatar_in_git - Gitlab::Routing.url_helpers.namespace_project_avatar_url(namespace, self) - end + def avatar_url(**args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) || (Gitlab::Routing.url_helpers.namespace_project_avatar_url(namespace, self) if avatar_in_git) end # For compatibility with old code @@ -859,14 +867,6 @@ class Project < ActiveRecord::Base @repo_exists = false end - # Branches that are not _exactly_ matched by a protected branch. - def open_branches - exact_protected_branch_names = protected_branches.reject(&:wildcard?).map(&:name) - branch_names = repository.branches.map(&:name) - non_open_branch_names = Set.new(exact_protected_branch_names).intersection(Set.new(branch_names)) - repository.branches.reject { |branch| non_open_branch_names.include? branch.name } - end - def root_ref?(branch) repository.root_ref == branch end @@ -881,16 +881,8 @@ class Project < ActiveRecord::Base Gitlab::UrlSanitizer.new("#{web_url}.git", credentials: credentials).full_url end - # Check if current branch name is marked as protected in the system - def protected_branch?(branch_name) - return true if empty_repo? && default_branch_protected? - - @protected_branches ||= self.protected_branches.to_a - ProtectedBranch.matching(branch_name, protected_branches: @protected_branches).present? - end - def user_can_push_to_empty_repo?(user) - !default_branch_protected? || team.max_member_access(user.id) > Gitlab::Access::DEVELOPER + !ProtectedBranch.default_branch_protected? || team.max_member_access(user.id) > Gitlab::Access::DEVELOPER end def forked? @@ -911,10 +903,10 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present" + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - # we currently doesn't support renaming repository if it contains tags in container registry - raise StandardError.new('Project cannot be renamed, because tags are present in its container registry') + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') end if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) @@ -975,7 +967,7 @@ class Project < ActiveRecord::Base namespace: namespace.name, visibility_level: visibility_level, path_with_namespace: path_with_namespace, - default_branch: default_branch, + default_branch: default_branch } # Backward compatibility @@ -1089,25 +1081,21 @@ class Project < ActiveRecord::Base end def shared_runners - shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none + @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none end - def any_runners?(&block) - if runners.active.any?(&block) - return true - end + def active_shared_runners + @active_shared_runners ||= shared_runners.active + end - shared_runners.active.any?(&block) + def any_runners?(&block) + active_runners.any?(&block) || active_shared_runners.any?(&block) end def valid_runners_token?(token) self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - def build_coverage_enabled? - build_coverage_regex.present? - end - def build_timeout_in_minutes build_timeout / 60 end @@ -1200,8 +1188,9 @@ class Project < ActiveRecord::Base end end + # Lazy loading of the `pipeline_status` attribute def pipeline_status - @pipeline_status ||= Ci::PipelineStatus.load_for_project(self) + @pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self) end def mark_import_as_failed(error_message) @@ -1261,7 +1250,7 @@ class Project < ActiveRecord::Base ] if container_registry_enabled? - variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true } + variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_url, public: true } end variables @@ -1287,6 +1276,9 @@ class Project < ActiveRecord::Base else update_attribute(name, value) end + + rescue ActiveRecord::RecordNotSaved => e + handle_update_attribute_error(e, value) end def pushes_since_gc @@ -1331,6 +1323,14 @@ class Project < ActiveRecord::Base namespace_id_changed? end + def default_merge_request_target + if forked_from_project&.merge_requests_enabled? + forked_from_project + else + self + end + end + alias_method :name_with_namespace, :full_name alias_method :human_name, :full_name alias_method :path_with_namespace, :full_path @@ -1357,11 +1357,6 @@ class Project < ActiveRecord::Base "projects/#{id}/pushes_since_gc" end - def default_branch_protected? - current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || - current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE - end - # Similar to the normal callbacks that hook into the life cycle of an # Active Record object, you can also define callbacks that get triggered # when you add an object to an association collection. If any of these @@ -1394,4 +1389,27 @@ class Project < ActiveRecord::Base Project.unscoped.where(pending_delete: true).find_by_full_path(path_with_namespace) end + + ## + # This method is here because of support for legacy container repository + # which has exactly the same path like project does, but which might not be + # persisted in `container_repositories` table. + # + def has_root_container_repository_tags? + return false unless Gitlab.config.registry.enabled + + ContainerRepository.build_root_repository(self).has_tags? + end + + def handle_update_attribute_error(ex, value) + if ex.message.start_with?('Failed to replace') + if value.respond_to?(:each) + invalid = value.detect(&:invalid?) + + raise ex, ([ex.message] + invalid.errors.full_messages).join(' ') if invalid + end + end + + raise ex + end end diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 400020ee04a..3f5b3eb159b 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -52,7 +52,7 @@ class BambooService < CiService placeholder: 'Bamboo build plan key like KEY' }, { type: 'text', name: 'username', placeholder: 'A user with API access, if applicable' }, - { type: 'password', name: 'password' }, + { type: 'password', name: 'password' } ] end diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index 86d271a3f69..e2ad586aea7 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -2,11 +2,23 @@ require 'slack-notifier' module ChatMessage class BaseMessage + attr_reader :markdown + attr_reader :user_name + attr_reader :user_avatar + attr_reader :project_name + attr_reader :project_url + def initialize(params) - raise NotImplementedError + @markdown = params[:markdown] || false + @project_name = params.dig(:project, :path_with_namespace) || params[:project_name] + @project_url = params.dig(:project, :web_url) || params[:project_url] + @user_name = params.dig(:user, :username) || params[:user_name] + @user_avatar = params.dig(:user, :avatar_url) || params[:user_avatar] end def pretext + return message if markdown + format(message) end @@ -17,6 +29,10 @@ module ChatMessage raise NotImplementedError end + def activity + raise NotImplementedError + end + private def message @@ -34,5 +50,16 @@ module ChatMessage def link(text, url) "[#{text}](#{url})" end + + def pretty_duration(seconds) + parse_string = + if duration < 1.hour + '%M:%S' + else + '%H:%M:%S' + end + + Time.at(seconds).utc.strftime(parse_string) + end end end diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index 791e5b0cec7..4b9a2b1e1f3 100644 --- a/app/models/project_services/chat_message/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -1,9 +1,6 @@ module ChatMessage class IssueMessage < BaseMessage - attr_reader :user_name attr_reader :title - attr_reader :project_name - attr_reader :project_url attr_reader :issue_iid attr_reader :issue_url attr_reader :action @@ -11,9 +8,7 @@ module ChatMessage attr_reader :description def initialize(params) - @user_name = params[:user][:username] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) @@ -27,15 +22,24 @@ module ChatMessage def attachments return [] unless opened_issue? + return description if markdown description_message end + def activity + { + title: "Issue #{state} by #{user_name}", + subtitle: "in #{project_link}", + text: issue_link, + image: user_avatar + } + end + private def message - case state - when "opened" + if state == 'opened' "[#{project_link}] Issue #{state} by #{user_name}" else "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" @@ -64,7 +68,7 @@ module ChatMessage end def issue_title - "##{issue_iid} #{title}" + "#{Issue.reference_prefix}#{issue_iid} #{title}" end end end diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index 5e5efca7bec..7d0de81cdf0 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -1,36 +1,36 @@ module ChatMessage class MergeMessage < BaseMessage - attr_reader :user_name - attr_reader :project_name - attr_reader :project_url - attr_reader :merge_request_id + attr_reader :merge_request_iid attr_reader :source_branch attr_reader :target_branch attr_reader :state attr_reader :title def initialize(params) - @user_name = params[:user][:username] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) - @merge_request_id = obj_attr[:iid] + @merge_request_iid = obj_attr[:iid] @source_branch = obj_attr[:source_branch] @target_branch = obj_attr[:target_branch] @state = obj_attr[:state] @title = format_title(obj_attr[:title]) end - def pretext - format(message) - end - def attachments [] end + def activity + { + title: "Merge Request #{state} by #{user_name}", + subtitle: "in #{project_link}", + text: merge_request_link, + image: user_avatar + } + end + private def format_title(title) @@ -50,11 +50,15 @@ module ChatMessage end def merge_request_link - link("merge request !#{merge_request_id}", merge_request_url) + link(merge_request_title, merge_request_url) + end + + def merge_request_title + "#{MergeRequest.reference_prefix}#{merge_request_iid} #{title}" end def merge_request_url - "#{project_url}/merge_requests/#{merge_request_id}" + "#{project_url}/merge_requests/#{merge_request_iid}" end end end diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index 552113bac29..2da4c244229 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -1,70 +1,74 @@ module ChatMessage class NoteMessage < BaseMessage - attr_reader :message - attr_reader :user_name - attr_reader :project_name - attr_reader :project_url attr_reader :note attr_reader :note_url + attr_reader :title + attr_reader :target def initialize(params) - params = HashWithIndifferentAccess.new(params) - @user_name = params[:user][:username] - @project_name = params[:project_name] - @project_url = params[:project_url] + super + params = HashWithIndifferentAccess.new(params) obj_attr = params[:object_attributes] - obj_attr = HashWithIndifferentAccess.new(obj_attr) @note = obj_attr[:note] @note_url = obj_attr[:url] - noteable_type = obj_attr[:noteable_type] - - case noteable_type - when "Commit" - create_commit_note(HashWithIndifferentAccess.new(params[:commit])) - when "Issue" - create_issue_note(HashWithIndifferentAccess.new(params[:issue])) - when "MergeRequest" - create_merge_note(HashWithIndifferentAccess.new(params[:merge_request])) - when "Snippet" - create_snippet_note(HashWithIndifferentAccess.new(params[:snippet])) - end + @target, @title = case obj_attr[:noteable_type] + when "Commit" + create_commit_note(params[:commit]) + when "Issue" + create_issue_note(params[:issue]) + when "MergeRequest" + create_merge_note(params[:merge_request]) + when "Snippet" + create_snippet_note(params[:snippet]) + end end def attachments + return note if markdown + description_message end + def activity + { + title: "#{user_name} #{link('commented on ' + target, note_url)}", + subtitle: "in #{project_link}", + text: formatted_title, + image: user_avatar + } + end + private + def message + "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*" + end + def format_title(title) title.lines.first.chomp end - def create_commit_note(commit) - commit_sha = commit[:id] - commit_sha = Commit.truncate_sha(commit_sha) - commented_on_message( - "commit #{commit_sha}", - format_title(commit[:message])) + def formatted_title + format_title(title) end def create_issue_note(issue) - commented_on_message( - "issue ##{issue[:iid]}", - format_title(issue[:title])) + ["issue #{Issue.reference_prefix}#{issue[:iid]}", issue[:title]] + end + + def create_commit_note(commit) + commit_sha = Commit.truncate_sha(commit[:id]) + + ["commit #{commit_sha}", commit[:message]] end def create_merge_note(merge_request) - commented_on_message( - "merge request !#{merge_request[:iid]}", - format_title(merge_request[:title])) + ["merge request #{MergeRequest.reference_prefix}#{merge_request[:iid]}", merge_request[:title]] end def create_snippet_note(snippet) - commented_on_message( - "snippet ##{snippet[:id]}", - format_title(snippet[:title])) + ["snippet #{Snippet.reference_prefix}#{snippet[:id]}", snippet[:title]] end def description_message @@ -74,9 +78,5 @@ module ChatMessage def project_link link(project_name, project_url) end - - def commented_on_message(target, title) - @message = "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{title}*" - end end end diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 210027565a8..3edc395033c 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -1,19 +1,22 @@ module ChatMessage class PipelineMessage < BaseMessage - attr_reader :ref_type, :ref, :status, :project_name, :project_url, - :user_name, :duration, :pipeline_id + attr_reader :ref_type + attr_reader :ref + attr_reader :status + attr_reader :duration + attr_reader :pipeline_id def initialize(data) + super + + @user_name = data.dig(:user, :name) || 'API' + pipeline_attributes = data[:object_attributes] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] - @duration = pipeline_attributes[:duration] + @duration = pipeline_attributes[:duration].to_i @pipeline_id = pipeline_attributes[:id] - - @project_name = data[:project][:path_with_namespace] - @project_url = data[:project][:web_url] - @user_name = (data[:user] && data[:user][:name]) || 'API' end def pretext @@ -25,17 +28,24 @@ module ChatMessage end def attachments + return message if markdown + [{ text: format(message), color: attachment_color }] end + def activity + { + title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status}", + subtitle: "in #{project_link}", + text: "in #{pretty_duration(duration)}", + image: user_avatar || '' + } + end + private def message - "#{project_link}: Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}" - end - - def format(string) - Slack::Notifier::LinkFormatter.format(string) + "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status} in #{pretty_duration(duration)}" end def humanized_status @@ -60,7 +70,7 @@ module ChatMessage end def branch_link - "[#{ref}](#{branch_url})" + "`[#{ref}](#{branch_url})`" end def project_link diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index 2d73b71ec37..04a59d559ca 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -3,33 +3,43 @@ module ChatMessage attr_reader :after attr_reader :before attr_reader :commits - attr_reader :project_name - attr_reader :project_url attr_reader :ref attr_reader :ref_type - attr_reader :user_name def initialize(params) + super + @after = params[:after] @before = params[:before] @commits = params.fetch(:commits, []) - @project_name = params[:project_name] - @project_url = params[:project_url] @ref_type = Gitlab::Git.tag_ref?(params[:ref]) ? 'tag' : 'branch' @ref = Gitlab::Git.ref_name(params[:ref]) - @user_name = params[:user_name] - end - - def pretext - format(message) end def attachments return [] if new_branch? || removed_branch? + return commit_messages if markdown commit_message_attachments end + def activity + action = if new_branch? + "created" + elsif removed_branch? + "removed" + else + "pushed to" + end + + { + title: "#{user_name} #{action} #{ref_type}", + subtitle: "in #{project_link}", + text: compare_link, + image: user_avatar + } + end + private def message @@ -51,7 +61,7 @@ module ChatMessage end def removed_branch_message - "#{user_name} removed #{ref_type} #{ref} from #{project_link}" + "#{user_name} removed #{ref_type} `#{ref}` from #{project_link}" end def push_message @@ -59,7 +69,7 @@ module ChatMessage end def commit_messages - commits.map { |commit| compose_commit_message(commit) }.join("\n") + commits.map { |commit| compose_commit_message(commit) }.join("\n\n") end def commit_message_attachments @@ -92,7 +102,7 @@ module ChatMessage end def branch_link - "[#{ref}](#{branch_url})" + "`[#{ref}](#{branch_url})`" end def project_link diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index 134083e4504..a139a8ee727 100644 --- a/app/models/project_services/chat_message/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -1,17 +1,12 @@ module ChatMessage class WikiPageMessage < BaseMessage - attr_reader :user_name attr_reader :title - attr_reader :project_name - attr_reader :project_url attr_reader :wiki_page_url attr_reader :action attr_reader :description def initialize(params) - @user_name = params[:user][:username] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) @@ -29,9 +24,20 @@ module ChatMessage end def attachments + return description if markdown + description_message end + def activity + { + title: "#{user_name} #{action} #{wiki_page_link}", + subtitle: "in #{project_link}", + text: title, + image: user_avatar + } + end + private def message diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 75834103db5..779ef54cfcb 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -39,7 +39,7 @@ class ChatNotificationService < Service { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}" }, { type: 'text', name: 'username', placeholder: 'e.g. GitLab' }, { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'checkbox', name: 'notify_only_default_branch' }, + { type: 'checkbox', name: 'notify_only_default_branch' } ] end @@ -49,10 +49,7 @@ class ChatNotificationService < Service object_kind = data[:object_kind] - data = data.merge( - project_url: project_url, - project_name: project_name - ) + data = custom_data(data) # WebHook events often have an 'update' event that follows a 'open' or # 'close' action. Ignore update events for now to prevent duplicate @@ -68,8 +65,7 @@ class ChatNotificationService < Service opts[:channel] = channel_name if channel_name opts[:username] = username if username - notifier = Slack::Notifier.new(webhook, opts) - notifier.ping(message.pretext, attachments: message.attachments, fallback: message.fallback) + return false unless notify(message, opts) true end @@ -92,6 +88,18 @@ class ChatNotificationService < Service private + def notify(message, opts) + Slack::Notifier.new(webhook, opts).ping( + message.pretext, + attachments: message.attachments, + fallback: message.fallback + ) + end + + def custom_data(data) + data.merge(project_url: project_url, project_name: project_name) + end + def get_message(object_kind, data) case object_kind when "push", "tag_push" @@ -142,7 +150,7 @@ class ChatNotificationService < Service def notify_for_ref?(data) return true if data[:object_attributes][:tag] - return true unless notify_only_default_branch + return true unless notify_only_default_branch? data[:object_attributes][:ref] == project.default_branch end diff --git a/app/models/project_services/emails_on_push_service.rb b/app/models/project_services/emails_on_push_service.rb index f4f913ee0b6..1a236e232f9 100644 --- a/app/models/project_services/emails_on_push_service.rb +++ b/app/models/project_services/emails_on_push_service.rb @@ -47,7 +47,7 @@ class EmailsOnPushService < Service help: "Send notifications from the committer's email address if the domain is part of the domain GitLab is running on (e.g. #{domains})." }, { type: 'checkbox', name: 'disable_diffs', title: "Disable code diffs", help: "Don't include possibly sensitive code diffs in notification body." }, - { type: 'textarea', name: 'recipients', placeholder: 'Emails separated by whitespace' }, + { type: 'textarea', name: 'recipients', placeholder: 'Emails separated by whitespace' } ] end end diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb index bdf6fa6a586..b4d7c977ce4 100644 --- a/app/models/project_services/external_wiki_service.rb +++ b/app/models/project_services/external_wiki_service.rb @@ -19,7 +19,7 @@ class ExternalWikiService < Service def fields [ - { type: 'text', name: 'external_wiki_url', placeholder: 'The URL of the external Wiki' }, + { type: 'text', name: 'external_wiki_url', placeholder: 'The URL of the external Wiki' } ] end diff --git a/app/models/project_services/flowdock_service.rb b/app/models/project_services/flowdock_service.rb index 10a13c3fbdc..2a05d757eb4 100644 --- a/app/models/project_services/flowdock_service.rb +++ b/app/models/project_services/flowdock_service.rb @@ -37,7 +37,7 @@ class FlowdockService < Service repo: project.repository.path_to_repo, repo_url: "#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}", commit_url: "#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}/commit/%s", - diff_url: "#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}/compare/%s...%s", + diff_url: "#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}/compare/%s...%s" ) end end diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 8b181221bb0..c19fed339ba 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -41,7 +41,7 @@ class HipchatService < Service placeholder: 'Leave blank for default (v2)' }, { type: 'text', name: 'server', placeholder: 'Leave blank for default. https://hipchat.example.com' }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' } ] end diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb index c62bb4fa120..a51d43adcb9 100644 --- a/app/models/project_services/irker_service.rb +++ b/app/models/project_services/irker_service.rb @@ -58,7 +58,7 @@ class IrkerService < Service ' want to use a password, you have to omit the "#" on the channel). If you ' \ ' specify a default IRC URI to prepend before each recipient, you can just ' \ ' give a channel name.' }, - { type: 'checkbox', name: 'colorize_messages' }, + { type: 'checkbox', name: 'colorize_messages' } ] end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index eef403dba92..f388773efee 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -62,7 +62,7 @@ class JiraService < IssueTrackerService def help "You need to configure JIRA before enabling this service. For more details read the - [JIRA service documentation](#{help_page_url('project_services/jira')})." + [JIRA service documentation](#{help_page_url('user/project/integrations/jira')})." end def title @@ -91,7 +91,7 @@ class JiraService < IssueTrackerService { type: 'text', name: 'project_key', placeholder: 'Project Key' }, { type: 'text', name: 'username', placeholder: '' }, { type: 'password', name: 'password', placeholder: '' }, - { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' } + { type: 'text', name: 'jira_issue_transition_id', placeholder: '' } ] end @@ -149,7 +149,7 @@ class JiraService < IssueTrackerService data = { user: { name: author.name, - url: resource_url(user_path(author)), + url: resource_url(user_path(author)) }, project: { name: self.project.path_with_namespace, diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 02fbd5497fa..b2494a0be6e 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -22,22 +22,21 @@ class KubernetesService < DeploymentService with_options presence: true, if: :activated? do validates :api_url, url: true validates :token - - validates :namespace, - format: { - with: Gitlab::Regex.kubernetes_namespace_regex, - message: Gitlab::Regex.kubernetes_namespace_regex_message, - }, - length: 1..63 end + validates :namespace, + allow_blank: true, + length: 1..63, + if: :activated?, + format: { + with: Gitlab::Regex.kubernetes_namespace_regex, + message: Gitlab::Regex.kubernetes_namespace_regex_message + } + after_save :clear_reactive_cache! def initialize_properties - if properties.nil? - self.properties = {} - self.namespace = "#{project.path}-#{project.id}" if project.present? - end + self.properties = {} if properties.nil? end def title @@ -62,7 +61,7 @@ class KubernetesService < DeploymentService { type: 'text', name: 'namespace', title: 'Kubernetes namespace', - placeholder: 'Kubernetes namespace' }, + placeholder: namespace_placeholder }, { type: 'text', name: 'api_url', title: 'API URL', @@ -74,7 +73,7 @@ class KubernetesService < DeploymentService { type: 'textarea', name: 'ca_pem', title: 'Custom CA bundle', - placeholder: 'Certificate Authority bundle (PEM format)' }, + placeholder: 'Certificate Authority bundle (PEM format)' } ] end @@ -92,7 +91,7 @@ class KubernetesService < DeploymentService variables = [ { key: 'KUBE_URL', value: api_url, public: true }, { key: 'KUBE_TOKEN', value: token, public: false }, - { key: 'KUBE_NAMESPACE', value: namespace, public: true } + { key: 'KUBE_NAMESPACE', value: namespace_variable, public: true } ] if ca_pem.present? @@ -135,8 +134,26 @@ class KubernetesService < DeploymentService { pods: pods } end + TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze + private + def namespace_placeholder + default_namespace || TEMPLATE_PLACEHOLDER + end + + def namespace_variable + if namespace.present? + namespace + else + default_namespace + end + end + + def default_namespace + "#{project.path}-#{project.id}" if project.present? + end + def build_kubeclient!(api_path: 'api', api_version: 'v1') raise "Incomplete settings" unless api_url && namespace && token diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb new file mode 100644 index 00000000000..2facff53e26 --- /dev/null +++ b/app/models/project_services/microsoft_teams_service.rb @@ -0,0 +1,56 @@ +class MicrosoftTeamsService < ChatNotificationService + def title + 'Microsoft Teams Notification' + end + + def description + 'Receive event notifications in Microsoft Teams' + end + + def self.to_param + 'microsoft_teams' + end + + def help + 'This service sends notifications about projects events to Microsoft Teams channels.<br /> + To set up this service: + <ol> + <li><a href="https://msdn.microsoft.com/en-us/microsoft-teams/connectors">Getting started with 365 Office Connectors For Microsoft Teams</a>.</li> + <li>Paste the <strong>Webhook URL</strong> into the field below.</li> + <li>Select events below to enable notifications.</li> + </ol>' + end + + def webhook_placeholder + 'https://outlook.office.com/webhook/…' + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}" }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'checkbox', name: 'notify_only_default_branch' } + ] + end + + private + + def notify(message, opts) + MicrosoftTeams::Notifier.new(webhook).ping( + title: message.project_name, + pretext: message.pretext, + activity: message.activity, + attachments: message.attachments + ) + end + + def custom_data(data) + super(data).merge(markdown: true) + end +end diff --git a/app/models/project_services/mock_ci_service.rb b/app/models/project_services/mock_ci_service.rb index a8d581a1f67..546b6e0a498 100644 --- a/app/models/project_services/mock_ci_service.rb +++ b/app/models/project_services/mock_ci_service.rb @@ -21,7 +21,7 @@ class MockCiService < CiService [ { type: 'text', name: 'mock_service_url', - placeholder: 'http://localhost:4004' }, + placeholder: 'http://localhost:4004' } ] end diff --git a/app/models/project_services/mock_deployment_service.rb b/app/models/project_services/mock_deployment_service.rb new file mode 100644 index 00000000000..59a3811ce5d --- /dev/null +++ b/app/models/project_services/mock_deployment_service.rb @@ -0,0 +1,18 @@ +class MockDeploymentService < DeploymentService + def title + 'Mock deployment' + end + + def description + 'Mock deployment service' + end + + def self.to_param + 'mock_deployment' + end + + # No terminals support + def terminals(environment) + [] + end +end diff --git a/app/models/project_services/mock_monitoring_service.rb b/app/models/project_services/mock_monitoring_service.rb new file mode 100644 index 00000000000..dd04e04e198 --- /dev/null +++ b/app/models/project_services/mock_monitoring_service.rb @@ -0,0 +1,17 @@ +class MockMonitoringService < MonitoringService + def title + 'Mock monitoring' + end + + def description + 'Mock monitoring service' + end + + def self.to_param + 'mock_monitoring' + end + + def metrics(environment) + JSON.parse(File.read(Rails.root + 'spec/fixtures/metrics.json')) + end +end diff --git a/app/models/project_services/monitoring_service.rb b/app/models/project_services/monitoring_service.rb index ea585721e8f..ee9cd78327a 100644 --- a/app/models/project_services/monitoring_service.rb +++ b/app/models/project_services/monitoring_service.rb @@ -9,8 +9,11 @@ class MonitoringService < Service %w() end - # Environments have a number of metrics - def metrics(environment) + def environment_metrics(environment) + raise NotImplementedError + end + + def deployment_metrics(deployment) raise NotImplementedError end end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb index ac617f409d9..f824171ad09 100644 --- a/app/models/project_services/pipelines_email_service.rb +++ b/app/models/project_services/pipelines_email_service.rb @@ -55,7 +55,7 @@ class PipelinesEmailService < Service name: 'recipients', placeholder: 'Emails separated by comma' }, { type: 'checkbox', - name: 'notify_only_broken_pipelines' }, + name: 'notify_only_broken_pipelines' } ] end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 6854d2243d7..ec72cb6856d 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -1,7 +1,6 @@ class PrometheusService < MonitoringService - include ReactiveCaching + include ReactiveService - self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] } self.reactive_cache_lease_timeout = 30.seconds self.reactive_cache_refresh_interval = 30.seconds self.reactive_cache_lifetime = 1.minute @@ -64,37 +63,31 @@ class PrometheusService < MonitoringService { success: false, result: err } end - def metrics(environment) - with_reactive_cache(environment.slug) do |data| - data - end + def environment_metrics(environment) + with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &:itself) + end + + def deployment_metrics(deployment) + metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &:itself) + metrics&.merge(deployment_time: created_at.to_i) || {} end # Cache metrics for specific environment - def calculate_reactive_cache(environment_slug) + def calculate_reactive_cache(query_class_name, *args) return unless active? && project && !project.pending_delete? - memory_query = %{(sum(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / count(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"})) /1024/1024} - cpu_query = %{sum(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) / count(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}) * 100} + metrics = Kernel.const_get(query_class_name).new(client).query(*args) { success: true, - metrics: { - # Average Memory used in MB - memory_values: client.query_range(memory_query, start: 8.hours.ago), - memory_current: client.query(memory_query), - # Average CPU Utilization - cpu_values: client.query_range(cpu_query, start: 8.hours.ago), - cpu_current: client.query(cpu_query) - }, + metrics: metrics, last_update: Time.now.utc } - rescue Gitlab::PrometheusError => err { success: false, result: err.message } end def client - @prometheus ||= Gitlab::Prometheus.new(api_url: api_url) + @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url) end end diff --git a/app/models/project_services/pushover_service.rb b/app/models/project_services/pushover_service.rb index 3e618a8dbf1..fc29a5277bb 100644 --- a/app/models/project_services/pushover_service.rb +++ b/app/models/project_services/pushover_service.rb @@ -55,7 +55,7 @@ class PushoverService < Service ['Pushover Echo (long)', 'echo'], ['Up Down (long)', 'updown'], ['None (silent)', 'none'] - ] }, + ] } ] end diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index cbaffb8ce48..b16beb406b9 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -55,7 +55,7 @@ class TeamcityService < CiService placeholder: 'Build configuration ID' }, { type: 'text', name: 'username', placeholder: 'A user with permissions to trigger a manual build' }, - { type: 'password', name: 'password' }, + { type: 'password', name: 'password' } ] end @@ -78,7 +78,7 @@ class TeamcityService < CiService auth = { username: username, - password: password, + password: password } branch = Gitlab::Git.ref_name(data[:ref]) diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 6d6644053f8..543b9b293e0 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -50,8 +50,8 @@ class ProjectTeam end def add_users(users, access_level, current_user: nil, expires_at: nil) - ProjectMember.add_users_to_projects( - [project.id], + ProjectMember.add_users( + project, users, access_level, current_user: current_user, diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 70eef359cdd..189c106b70b 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -183,6 +183,6 @@ class ProjectWiki end def update_project_activity - @project.touch(:last_activity_at) + @project.touch(:last_activity_at, :last_repository_updated_at) end end diff --git a/app/models/protectable_dropdown.rb b/app/models/protectable_dropdown.rb new file mode 100644 index 00000000000..122fbce257d --- /dev/null +++ b/app/models/protectable_dropdown.rb @@ -0,0 +1,33 @@ +class ProtectableDropdown + def initialize(project, ref_type) + @project = project + @ref_type = ref_type + end + + # Tags/branches which are yet to be individually protected + def protectable_ref_names + @protectable_ref_names ||= ref_names - non_wildcard_protected_ref_names + end + + def hash + protectable_ref_names.map { |ref_name| { text: ref_name, id: ref_name, title: ref_name } } + end + + private + + def refs + @project.repository.public_send(@ref_type) + end + + def ref_names + refs.map(&:name) + end + + def protections + @project.public_send("protected_#{@ref_type}") + end + + def non_wildcard_protected_ref_names + protections.reject(&:wildcard?).map(&:name) + end +end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 39e979ef15b..28b7d5ad072 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -1,9 +1,6 @@ class ProtectedBranch < ActiveRecord::Base include Gitlab::ShellAdapter - - belongs_to :project - validates :name, presence: true - validates :project, presence: true + include ProtectedRef has_many :merge_access_levels, dependent: :destroy has_many :push_access_levels, dependent: :destroy @@ -14,54 +11,15 @@ class ProtectedBranch < ActiveRecord::Base accepts_nested_attributes_for :push_access_levels accepts_nested_attributes_for :merge_access_levels - def commit - project.commit(self.name) - end - - # Returns all protected branches that match the given branch name. - # This realizes all records from the scope built up so far, and does - # _not_ return a relation. - # - # This method optionally takes in a list of `protected_branches` to search - # through, to avoid calling out to the database. - def self.matching(branch_name, protected_branches: nil) - (protected_branches || all).select { |protected_branch| protected_branch.matches?(branch_name) } - end - - # Returns all branches (among the given list of branches [`Gitlab::Git::Branch`]) - # that match the current protected branch. - def matching(branches) - branches.select { |branch| self.matches?(branch.name) } - end - - # Checks if the protected branch matches the given branch name. - def matches?(branch_name) - return false if self.name.blank? - - exact_match?(branch_name) || wildcard_match?(branch_name) - end - - # Checks if this protected branch contains a wildcard - def wildcard? - self.name && self.name.include?('*') - end - - protected - - def exact_match?(branch_name) - self.name == branch_name - end + # Check if branch name is marked as protected in the system + def self.protected?(project, ref_name) + return true if project.empty_repo? && default_branch_protected? - def wildcard_match?(branch_name) - wildcard_regex === branch_name + self.matching(ref_name, protected_refs: project.protected_branches).present? end - def wildcard_regex - @wildcard_regex ||= begin - name = self.name.gsub('*', 'STAR_DONT_ESCAPE') - quoted_name = Regexp.quote(name) - regex_string = quoted_name.gsub('STAR_DONT_ESCAPE', '.*?') - /\A#{regex_string}\z/ - end + def self.default_branch_protected? + current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_FULL || + current_application_settings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE end end diff --git a/app/models/protected_branch/merge_access_level.rb b/app/models/protected_branch/merge_access_level.rb index 771e3376613..e8d35ac326f 100644 --- a/app/models/protected_branch/merge_access_level.rb +++ b/app/models/protected_branch/merge_access_level.rb @@ -1,13 +1,3 @@ class ProtectedBranch::MergeAccessLevel < ActiveRecord::Base include ProtectedBranchAccess - - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER] } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters" - }.with_indifferent_access - end end diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index 14610cb42b7..7a2e9e5ec5d 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -1,21 +1,3 @@ class ProtectedBranch::PushAccessLevel < ActiveRecord::Base include ProtectedBranchAccess - - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::NO_ACCESS] } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters", - Gitlab::Access::NO_ACCESS => "No one" - }.with_indifferent_access - end - - def check_access(user) - return false if access_level == Gitlab::Access::NO_ACCESS - - super - end end diff --git a/app/models/protected_ref_matcher.rb b/app/models/protected_ref_matcher.rb new file mode 100644 index 00000000000..d970f2b01fc --- /dev/null +++ b/app/models/protected_ref_matcher.rb @@ -0,0 +1,54 @@ +class ProtectedRefMatcher + def initialize(protected_ref) + @protected_ref = protected_ref + end + + # Returns all protected refs that match the given ref name. + # This checks all records from the scope built up so far, and does + # _not_ return a relation. + # + # This method optionally takes in a list of `protected_refs` to search + # through, to avoid calling out to the database. + def self.matching(type, ref_name, protected_refs: nil) + (protected_refs || type.all).select { |protected_ref| protected_ref.matches?(ref_name) } + end + + # Returns all branches/tags (among the given list of refs [`Gitlab::Git::Branch`]) + # that match the current protected ref. + def matching(refs) + refs.select { |ref| @protected_ref.matches?(ref.name) } + end + + # Checks if the protected ref matches the given ref name. + def matches?(ref_name) + return false if @protected_ref.name.blank? + + exact_match?(ref_name) || wildcard_match?(ref_name) + end + + # Checks if this protected ref contains a wildcard + def wildcard? + @protected_ref.name && @protected_ref.name.include?('*') + end + + protected + + def exact_match?(ref_name) + @protected_ref.name == ref_name + end + + def wildcard_match?(ref_name) + return false unless wildcard? + + wildcard_regex === ref_name + end + + def wildcard_regex + @wildcard_regex ||= begin + name = @protected_ref.name.gsub('*', 'STAR_DONT_ESCAPE') + quoted_name = Regexp.quote(name) + regex_string = quoted_name.gsub('STAR_DONT_ESCAPE', '.*?') + /\A#{regex_string}\z/ + end + end +end diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb new file mode 100644 index 00000000000..83964095516 --- /dev/null +++ b/app/models/protected_tag.rb @@ -0,0 +1,14 @@ +class ProtectedTag < ActiveRecord::Base + include Gitlab::ShellAdapter + include ProtectedRef + + has_many :create_access_levels, dependent: :destroy + + validates :create_access_levels, length: { is: 1, message: "are restricted to a single instance per protected tag." } + + accepts_nested_attributes_for :create_access_levels + + def self.protected?(project, ref_name) + self.matching(ref_name, protected_refs: project.protected_tags).present? + end +end diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb new file mode 100644 index 00000000000..c7e1319719d --- /dev/null +++ b/app/models/protected_tag/create_access_level.rb @@ -0,0 +1,21 @@ +class ProtectedTag::CreateAccessLevel < ActiveRecord::Base + include ProtectedTagAccess + + validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::NO_ACCESS] } + + def self.human_access_levels + { + Gitlab::Access::MASTER => "Masters", + Gitlab::Access::DEVELOPER => "Developers + Masters", + Gitlab::Access::NO_ACCESS => "No one" + }.with_indifferent_access + end + + def check_access(user) + return false if access_level == Gitlab::Access::NO_ACCESS + + super + end +end diff --git a/app/models/readme_blob.rb b/app/models/readme_blob.rb new file mode 100644 index 00000000000..1863a08f1de --- /dev/null +++ b/app/models/readme_blob.rb @@ -0,0 +1,13 @@ +class ReadmeBlob < SimpleDelegator + attr_reader :repository + + def initialize(blob, repository) + @repository = repository + + super(blob) + end + + def rendered_markup + repository.rendered_readme + end +end diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb new file mode 100644 index 00000000000..99812bcde53 --- /dev/null +++ b/app/models/redirect_route.rb @@ -0,0 +1,12 @@ +class RedirectRoute < ActiveRecord::Base + belongs_to :source, polymorphic: true + + validates :source, presence: true + + validates :path, + length: { within: 1..255 }, + presence: true, + uniqueness: { case_sensitive: false } + + scope :matching_path_and_descendants, -> (path) { where('redirect_routes.path = ? OR redirect_routes.path LIKE ?', path, "#{sanitize_sql_like(path)}/%") } +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6ab04440ca8..07e0b3bae4f 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -2,9 +2,12 @@ require 'securerandom' class Repository include Gitlab::ShellAdapter + include RepositoryMirroring attr_accessor :path_with_namespace, :project + delegate :ref_name_for_sha, to: :raw_repository + CommitError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) @@ -14,9 +17,9 @@ class Repository # same name. The cache key used by those methods must also match method's # name. # - # For example, for entry `:readme` there's a method called `readme` which - # stores its data in the `readme` cache key. - CACHED_METHODS = %i(size commit_count readme version contribution_guide + # For example, for entry `:commit_count` there's a method called `commit_count` which + # stores its data in the `commit_count` cache key. + CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide changelog license_blob license_key gitignore koding_yml gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? empty? root_ref).freeze @@ -25,11 +28,10 @@ class Repository # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to # the corresponding methods to call for refreshing caches. METHOD_CACHES_FOR_FILE_TYPES = { - readme: :readme, + readme: :rendered_readme, changelog: :changelog, - license: %i(license_blob license_key), + license: %i(license_blob license_key license), contributing: :contribution_guide, - version: :version, gitignore: :gitignore, koding: :koding_yml, gitlab_ci: :gitlab_ci_yml, @@ -40,13 +42,13 @@ class Repository # variable. # # This only works for methods that do not take any arguments. - def self.cache_method(name, fallback: nil) + def self.cache_method(name, fallback: nil, memoize_only: false) original = :"_uncached_#{name}" alias_method(original, name) define_method(name) do - cache_method_output(name, fallback: fallback) { __send__(original) } + cache_method_output(name, fallback: fallback, memoize_only: memoize_only) { __send__(original) } end end @@ -58,13 +60,13 @@ class Repository def raw_repository return nil unless path_with_namespace - @raw_repository ||= Gitlab::Git::Repository.new(path_to_repo) + @raw_repository ||= initialize_raw_repository end # Return absolute path to repository def path_to_repo @path_to_repo ||= File.expand_path( - File.join(@project.repository_storage_path, path_with_namespace + ".git") + File.join(repository_storage_path, path_with_namespace + ".git") ) end @@ -106,7 +108,7 @@ class Repository offset: offset, after: after, before: before, - follow: path.present?, + follow: Array(path).length == 1, skip_merges: skip_merges } @@ -145,12 +147,7 @@ class Repository # may cause the branch to "disappear" erroneously or have the wrong SHA. # # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392 - raw_repo = - if fresh_repo - Gitlab::Git::Repository.new(path_to_repo) - else - raw_repository - end + raw_repo = fresh_repo ? initialize_raw_repository : raw_repository raw_repo.find_branch(name) end @@ -401,10 +398,6 @@ class Repository expire_tags_cache end - def before_import - expire_content_cache - end - # Runs code after the HEAD of a repository is changed. def after_change_head expire_method_caches(METHOD_CACHES_FOR_FILE_TYPES.keys) @@ -413,8 +406,6 @@ class Repository # Runs code after a repository has been forked/imported. def after_import expire_content_cache - expire_tags_cache - expire_branches_cache end # Runs code after a new commit has been pushed. @@ -459,7 +450,7 @@ class Repository def blob_at(sha, path) unless Gitlab::Git.blank_ref?(sha) - Blob.decorate(Gitlab::Git::Blob.find(self, sha, path)) + Blob.decorate(Gitlab::Git::Blob.find(self, sha, path), project) end rescue Gitlab::Git::Repository::NoRepository nil @@ -508,22 +499,14 @@ class Repository end end - def branch_names - branches.map(&:name) - end + delegate :branch_names, to: :raw_repository cache_method :branch_names, fallback: [] delegate :tag_names, to: :raw_repository cache_method :tag_names, fallback: [] - def branch_count - branches.size - end + delegate :branch_count, :tag_count, to: :raw_repository cache_method :branch_count, fallback: 0 - - def tag_count - raw_repository.rugged.tags.count - end cache_method :tag_count, fallback: 0 def avatar @@ -534,16 +517,15 @@ class Repository cache_method :avatar def readme - if head = tree(:head) - head.readme + if readme = tree(:head)&.readme + ReadmeBlob.new(readme, self) end end - cache_method :readme - def version - file_on_head(:version) + def rendered_readme + MarkupHelper.markup_unsafe(readme.name, readme.data, project: project) if readme end - cache_method :version + cache_method :rendered_readme def contribution_guide file_on_head(:contributing) @@ -567,6 +549,13 @@ class Repository end cache_method :license_key + def license + return unless license_key + + Licensee::License.new(license_key) + end + cache_method :license, memoize_only: true + def gitignore file_on_head(:gitignore) end @@ -660,22 +649,8 @@ class Repository "#{name}-#{highest_branch_id + 1}" end - # Remove archives older than 2 hours def branches_sorted_by(value) - case value - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - commit(b.dereferenced_target).committed_date <=> commit(a.dereferenced_target).committed_date - end - when 'updated_asc' - branches.sort do |a, b| - commit(a.dereferenced_target).committed_date <=> commit(b.dereferenced_target).committed_date - end - else - branches - end + raw_repository.local_branches(sort_by: value) end def tags_sorted_by(value) @@ -710,14 +685,6 @@ class Repository end end - def ref_name_for_sha(ref_path, sha) - args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - Gitlab::Popen.popen(args, path_to_repo).first.split.last - end - def refs_contains_sha(ref_type, sha) args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha}) names = Gitlab::Popen.popen(args, path_to_repo).first @@ -815,7 +782,7 @@ class Repository } options.merge!(get_committer_and_author(user, email: author_email, name: author_name)) - Rugged::Commit.create(rugged, options) + create_commit(options) end end # rubocop:enable Metrics/ParameterLists @@ -859,10 +826,10 @@ class Repository actual_options = options.merge( parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), + tree: merge_index.write_tree(rugged) ) - commit_id = Rugged::Commit.create(rugged, actual_options) + commit_id = create_commit(actual_options) merge_request.update(in_progress_merge_commit_sha: commit_id) commit_id end @@ -885,12 +852,11 @@ class Repository committer = user_to_committer(user) - Rugged::Commit.create(rugged, - message: commit.revert_message(user), - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) + create_commit(message: commit.revert_message(user), + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) end end @@ -909,16 +875,15 @@ class Repository committer = user_to_committer(user) - Rugged::Commit.create(rugged, - message: commit.message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) + create_commit(message: commit.message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) end end @@ -926,7 +891,7 @@ class Repository GitOperationService.new(user, self).with_branch(branch_name) do committer = user_to_committer(user) - Rugged::Commit.create(rugged, params.merge(author: committer, committer: committer)) + create_commit(params.merge(author: committer, committer: committer)) end end @@ -981,7 +946,13 @@ class Repository end def is_ancestor?(ancestor_id, descendant_id) - merge_base(ancestor_id, descendant_id) == ancestor_id + Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| + if is_enabled + raw_repository.is_ancestor?(ancestor_id, descendant_id) + else + merge_base_commit(ancestor_id, descendant_id) == ancestor_id + end + end end def empty_repo? @@ -1027,6 +998,23 @@ class Repository rugged.references.delete(tmp_ref) if tmp_ref end + def add_remote(name, url) + raw_repository.remote_add(name, url) + rescue Rugged::ConfigError + raw_repository.remote_update(name, url: url) + end + + def remove_remote(name) + raw_repository.remote_delete(name) + true + rescue Rugged::ConfigError + false + end + + def fetch_remote(remote, forced: false, no_tags: false) + gitlab_shell.fetch_remote(repository_storage_path, path_with_namespace, remote, forced: forced, no_tags: no_tags) + end + def fetch_ref(source_path, source_ref, target_ref) args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) Gitlab::Popen.popen(args, path_to_repo) @@ -1066,14 +1054,20 @@ class Repository # # key - The name of the key to cache the data in. # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, &block) + def cache_method_output(key, fallback: nil, memoize_only: false, &block) ivar = cache_instance_variable_name(key) if instance_variable_defined?(ivar) instance_variable_get(ivar) else begin - instance_variable_set(ivar, cache.fetch(key, &block)) + value = + if memoize_only + yield + else + cache.fetch(key, &block) + end + instance_variable_set(ivar, value) rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository # if e.g. HEAD or the entire repository doesn't exist we want to # gracefully handle this and not cache anything. @@ -1088,8 +1082,8 @@ class Repository def file_on_head(type) if head = tree(:head) - head.blobs.find do |file| - Gitlab::FileDetector.type_of(file.name) == type + head.blobs.find do |blob| + Gitlab::FileDetector.type_of(blob.path) == type end end end @@ -1144,4 +1138,18 @@ class Repository def repository_event(event, tags = {}) Gitlab::Metrics.add_event(event, { path: path_with_namespace }.merge(tags)) end + + def create_commit(params = {}) + params[:message].delete!("\r") + + Rugged::Commit.create(rugged, params) + end + + def repository_storage_path + @project.repository_storage_path + end + + def initialize_raw_repository + Gitlab::Git::Repository.new(project.repository_storage, path_with_namespace + '.git') + end end diff --git a/app/models/route.rb b/app/models/route.rb index 4b3efab5c3c..be77b8b51a5 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -8,29 +8,58 @@ class Route < ActiveRecord::Base presence: true, uniqueness: { case_sensitive: false } + after_create :delete_conflicting_redirects + after_update :delete_conflicting_redirects, if: :path_changed? + after_update :create_redirect_for_old_path after_update :rename_descendants scope :inside_path, -> (path) { where('routes.path LIKE ?', "#{sanitize_sql_like(path)}/%") } def rename_descendants - if path_changed? || name_changed? - descendants = self.class.inside_path(path_was) + return unless path_changed? || name_changed? - descendants.each do |route| - attributes = {} + descendant_routes = self.class.inside_path(path_was) - if path_changed? && route.path.present? - attributes[:path] = route.path.sub(path_was, path) - end + descendant_routes.each do |route| + attributes = {} - if name_changed? && name_was.present? && route.name.present? - attributes[:name] = route.name.sub(name_was, name) - end + if path_changed? && route.path.present? + attributes[:path] = route.path.sub(path_was, path) + end - # Note that update_columns skips validation and callbacks. - # We need this to avoid recursive call of rename_descendants method - route.update_columns(attributes) unless attributes.empty? + if name_changed? && name_was.present? && route.name.present? + attributes[:name] = route.name.sub(name_was, name) + end + + if attributes.present? + old_path = route.path + + # Callbacks must be run manually + route.update_columns(attributes.merge(updated_at: Time.now)) + + # We are not calling route.delete_conflicting_redirects here, in hopes + # of avoiding deadlocks. The parent (self, in this method) already + # called it, which deletes conflicts for all descendants. + route.create_redirect(old_path) if attributes[:path] end end end + + def delete_conflicting_redirects + conflicting_redirects.delete_all + end + + def conflicting_redirects + RedirectRoute.matching_path_and_descendants(path) + end + + def create_redirect(path) + RedirectRoute.create(source: source, path: path) + end + + private + + def create_redirect_for_old_path + create_redirect(path_was) if path_changed? + end end diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index f4bcb49b34d..0ae5864615a 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -5,10 +5,11 @@ class SentNotification < ActiveRecord::Base belongs_to :noteable, polymorphic: true belongs_to :recipient, class_name: "User" - validates :project, :recipient, :reply_key, presence: true - validates :reply_key, uniqueness: true + validates :project, :recipient, presence: true + validates :reply_key, presence: true, uniqueness: true validates :noteable_id, presence: true, unless: :for_commit? validates :commit_id, presence: true, if: :for_commit? + validates :in_reply_to_discussion_id, format: { with: /\A\h{40}\z/, allow_nil: true } validate :note_valid after_save :keep_around_commit @@ -22,9 +23,7 @@ class SentNotification < ActiveRecord::Base find_by(reply_key: reply_key) end - def record(noteable, recipient_id, reply_key, attrs = {}) - return unless reply_key - + def record(noteable, recipient_id, reply_key = self.reply_key, attrs = {}) noteable_id = nil commit_id = nil if noteable.is_a?(Commit) @@ -34,23 +33,20 @@ class SentNotification < ActiveRecord::Base end attrs.reverse_merge!( - project: noteable.project, - noteable_type: noteable.class.name, - noteable_id: noteable_id, - commit_id: commit_id, - recipient_id: recipient_id, - reply_key: reply_key + project: noteable.project, + recipient_id: recipient_id, + reply_key: reply_key, + + noteable_type: noteable.class.name, + noteable_id: noteable_id, + commit_id: commit_id ) create(attrs) end - def record_note(note, recipient_id, reply_key, attrs = {}) - if note.diff_note? - attrs[:note_type] = note.type - - attrs.merge!(note.diff_attributes) - end + def record_note(note, recipient_id, reply_key = self.reply_key, attrs = {}) + attrs[:in_reply_to_discussion_id] = note.discussion_id record(note.noteable, recipient_id, reply_key, attrs) end @@ -89,31 +85,45 @@ class SentNotification < ActiveRecord::Base self.reply_key end - def note_attributes - { - project: self.project, - author: self.recipient, - type: self.note_type, - noteable_type: self.noteable_type, - noteable_id: self.noteable_id, - commit_id: self.commit_id, - line_code: self.line_code, - position: self.position.to_json - } - end - - def create_note(note) - Notes::CreateService.new( - self.project, - self.recipient, - self.note_attributes.merge(note: note) - ).execute + def create_reply(message, dryrun: false) + klass = dryrun ? Notes::BuildService : Notes::CreateService + klass.new(self.project, self.recipient, reply_params.merge(note: message)).execute end private + def reply_params + attrs = { + noteable_type: self.noteable_type, + noteable_id: self.noteable_id, + commit_id: self.commit_id + } + + if self.in_reply_to_discussion_id.present? + attrs[:in_reply_to_discussion_id] = self.in_reply_to_discussion_id + else + # Remove in GitLab 10.0, when we will not support replying to SentNotifications + # that don't have `in_reply_to_discussion_id` anymore. + attrs.merge!( + type: self.note_type, + + # LegacyDiffNote + line_code: self.line_code, + + # DiffNote + position: self.position.to_json + ) + end + + attrs + end + def note_valid - Note.new(note_attributes.merge(note: "Test")).valid? + note = create_reply('Test', dryrun: true) + + unless note.valid? + self.errors.add(:base, "Note parameters are invalid: #{note.errors.full_messages.to_sentence}") + end end def keep_around_commit diff --git a/app/models/service.rb b/app/models/service.rb index e73f7e5d1a3..8916f88076e 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,7 @@ class Service < ActiveRecord::Base default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true - default_value_for :build_events, true + default_value_for :job_events, true default_value_for :pipeline_events, true default_value_for :wiki_page_events, true @@ -25,7 +25,8 @@ class Service < ActiveRecord::Base belongs_to :project, inverse_of: :services has_one :service_hook - validates :project_id, presence: true, unless: Proc.new { |service| service.template? } + validates :project_id, presence: true, unless: proc { |service| service.template? } + validates :type, presence: true scope :visible, -> { where.not(type: 'GitlabIssueTrackerService') } scope :issue_trackers, -> { where(category: 'issue_tracker') } @@ -39,7 +40,7 @@ class Service < ActiveRecord::Base scope :confidential_issue_hooks, -> { where(confidential_issues_events: true, active: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true, active: true) } scope :note_hooks, -> { where(note_events: true, active: true) } - scope :build_hooks, -> { where(build_events: true, active: true) } + scope :job_hooks, -> { where(job_events: true, active: true) } scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) } scope :external_issue_trackers, -> { issue_trackers.active.without_defaults } @@ -131,7 +132,7 @@ class Service < ActiveRecord::Base end def can_test? - !project.empty_repo? + true end # reason why service cannot be tested @@ -237,8 +238,11 @@ class Service < ActiveRecord::Base slack_slash_commands slack teamcity + microsoft_teams ] - service_names << 'mock_ci' if Rails.env.development? + if Rails.env.development? + service_names += %w[mock_ci mock_deployment mock_monitoring] + end service_names.sort_by(&:downcase) end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 30aca62499c..882e2fa0594 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -1,7 +1,7 @@ class Snippet < ActiveRecord::Base include Gitlab::VisibilityLevel - include Linguist::BlobHelper include CacheMarkdownField + include Noteable include Participable include Referable include Sortable @@ -12,6 +12,11 @@ class Snippet < ActiveRecord::Base cache_markdown_field :title, pipeline: :single_line cache_markdown_field :content + # Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with snippets. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10392/diffs#note_28719102 + alias_attribute :last_edited_at, :updated_at + alias_attribute :last_edited_by, :updated_by + # If file_name changes, it invalidates content alias_method :default_content_html_invalidator, :content_html_invalidated? def content_html_invalidated? @@ -86,47 +91,26 @@ class Snippet < ActiveRecord::Base ] end - def data - content + def blob + @blob ||= Blob.decorate(SnippetBlob.new(self), nil) end def hook_attrs attributes end - def size - 0 - end - def file_name super.to_s end - # alias for compatibility with blobs and highlighting - def path - file_name - end - - def name - file_name - end - def sanitized_file_name file_name.gsub(/[^a-zA-Z0-9_\-\.]+/, '') end - def mode - nil - end - def visibility_level_field :visibility_level end - def no_highlighting? - content.lines.count > 1000 - end - def notes_with_associations notes.includes(:author) end @@ -168,18 +152,5 @@ class Snippet < ActiveRecord::Base where(table[:content].matches(pattern)) end - - def accessible_to(user) - return are_public unless user.present? - return all if user.admin? - - where( - 'visibility_level IN (:visibility_levels) - OR author_id = :author_id - OR project_id IN (:project_ids)', - visibility_levels: [Snippet::PUBLIC, Snippet::INTERNAL], - author_id: user.id, - project_ids: user.authorized_projects.select(:id)) - end end end diff --git a/app/models/snippet_blob.rb b/app/models/snippet_blob.rb new file mode 100644 index 00000000000..fa5fa151607 --- /dev/null +++ b/app/models/snippet_blob.rb @@ -0,0 +1,31 @@ +class SnippetBlob + include BlobLike + + attr_reader :snippet + + def initialize(snippet) + @snippet = snippet + end + + delegate :id, to: :snippet + + def name + snippet.file_name + end + + alias_method :path, :name + + def size + data.bytesize + end + + def data + snippet.content + end + + def rendered_markup + return unless Gitlab::MarkupHelper.gitlab_markdown?(name) + + Banzai.render_field(snippet, :content) + end +end diff --git a/app/models/spam_log.rb b/app/models/spam_log.rb index 3b8b9833565..dd21ee15c6c 100644 --- a/app/models/spam_log.rb +++ b/app/models/spam_log.rb @@ -3,9 +3,9 @@ class SpamLog < ActiveRecord::Base validates :user, presence: true - def remove_user + def remove_user(deleted_by:) user.block - user.destroy + DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true, hard_delete: true) end def text diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 5cc66574941..b44f4fe000c 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -1,7 +1,7 @@ class SystemNoteMetadata < ActiveRecord::Base ICON_TYPES = %w[ - commit merge confidentiality status label assignee cross_reference - title time_tracking branch milestone discussion task moved + commit description merge confidential visible label assignee cross_reference + title time_tracking branch milestone discussion task moved opened closed merged ].freeze validates :note, presence: true diff --git a/app/models/todo.rb b/app/models/todo.rb index da3fa7277c2..b011001b235 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -84,6 +84,10 @@ class Todo < ActiveRecord::Base action == BUILD_FAILED end + def assigned? + action == ASSIGNED + end + def action_name ACTION_NAMES[action] end @@ -117,6 +121,14 @@ class Todo < ActiveRecord::Base end end + def self_added? + author == user + end + + def self_assigned? + assigned? && self_added? + end + private def keep_around_commit diff --git a/app/models/tree.rb b/app/models/tree.rb index fe148b0ec65..c89b8eca9be 100644 --- a/app/models/tree.rb +++ b/app/models/tree.rb @@ -40,10 +40,7 @@ class Tree readme_path = path == '/' ? readme_tree.name : File.join(path, readme_tree.name) - git_repo = repository.raw_repository - @readme = Gitlab::Git::Blob.find(git_repo, sha, readme_path) - @readme.load_all_data!(git_repo) - @readme + @readme = repository.blob_at(sha, readme_path) end def trees diff --git a/app/models/user.rb b/app/models/user.rb index cbd741f96ed..837ab78228b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -5,6 +5,7 @@ class User < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::CurrentSettings + include Avatarable include Referable include Sortable include CaseSensitivity @@ -23,6 +24,7 @@ class User < ActiveRecord::Base default_value_for :hide_no_password, false default_value_for :project_view, :files default_value_for :notified_of_own_activity, false + default_value_for :preferred_language, I18n.default_locale attr_encrypted :otp_secret, key: Gitlab::Application.secrets.otp_key_base, @@ -39,6 +41,17 @@ class User < ActiveRecord::Base devise :lockable, :recoverable, :rememberable, :trackable, :validatable, :omniauthable, :confirmable, :registerable + # Override Devise::Models::Trackable#update_tracked_fields! + # to limit database writes to at most once every hour + def update_tracked_fields!(request) + update_tracked_fields(request) + + lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i) + return unless lease.try_obtain + + save(validate: false) + end + attr_accessor :force_random_password # Virtual attribute for authenticating by either username or email @@ -89,7 +102,8 @@ class User < ActiveRecord::Base has_many :subscriptions, dependent: :destroy has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy - has_one :abuse_report, dependent: :destroy + has_one :abuse_report, dependent: :destroy, foreign_key: :user_id + has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" has_many :spam_logs, dependent: :destroy has_many :builds, dependent: :nullify, class_name: 'Ci::Build' has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' @@ -98,7 +112,8 @@ class User < ActiveRecord::Base has_many :award_emoji, dependent: :destroy has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id - has_many :assigned_issues, dependent: :nullify, foreign_key: :assignee_id, class_name: "Issue" + has_many :issue_assignees + has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # Issues that a user owns are expected to be moved to the "ghost" user before @@ -120,7 +135,7 @@ class User < ActiveRecord::Base presence: true, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: Gitlab::Database::MAX_INT_VALUE } validates :username, - namespace: true, + dynamic_path: true, presence: true, uniqueness: { case_sensitive: false } @@ -151,8 +166,13 @@ class User < ActiveRecord::Base enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos] # User's Project preference - # Note: When adding an option, it MUST go on the end of the array. - enum project_view: [:readme, :activity, :files] + # + # Note: When adding an option, it MUST go on the end of the hash with a + # number higher than the current max. We cannot move options and/or change + # their numbers. + # + # We skip 0 because this was used by an option that has since been removed. + enum project_view: { activity: 1, files: 2 } alias_attribute :private_token, :authentication_token @@ -196,7 +216,7 @@ class User < ActiveRecord::Base scope :admins, -> { where(admin: true) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :external, -> { where(external: true) } - scope :active, -> { with_state(:active) } + scope :active, -> { with_state(:active).non_internal } scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all } scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') } scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) } @@ -334,6 +354,11 @@ class User < ActiveRecord::Base find_by(id: Key.unscoped.select(:user_id).where(id: key_id)) end + def find_by_full_path(path, follow_redirects: false) + namespace = Namespace.for_user.find_by_full_path(path, follow_redirects: follow_redirects) + namespace&.owner + end + def reference_prefix '@' end @@ -356,6 +381,10 @@ class User < ActiveRecord::Base end end + def full_path + username + end + def self.internal_attributes [:ghost] end @@ -484,6 +513,14 @@ class User < ActiveRecord::Base Group.member_descendants(id) end + def all_expanded_groups + Group.member_hierarchy(id) + end + + def expanded_groups_requiring_two_factor_authentication + all_expanded_groups.where(require_two_factor_authentication: true) + end + def nested_groups_projects Project.joins(:namespace).where('namespaces.parent_id IS NOT NULL'). member_descendants(id) @@ -546,10 +583,6 @@ class User < ActiveRecord::Base authorized_projects(Gitlab::Access::REPORTER).non_archived.with_issues_enabled end - def is_admin? - admin - end - def require_ssh_key? keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh') end @@ -582,10 +615,6 @@ class User < ActiveRecord::Base name.split.first unless name.blank? end - def cared_merge_requests - MergeRequest.cared(self) - end - def projects_limit_left projects_limit - personal_projects.count end @@ -635,8 +664,10 @@ class User < ActiveRecord::Base end def fork_of(project) - links = ForkedProjectLink.where(forked_from_project_id: project, forked_to_project_id: personal_projects) - + links = ForkedProjectLink.where( + forked_from_project_id: project, + forked_to_project_id: personal_projects.unscope(:order) + ) if links.any? links.first.forked_to_project else @@ -759,12 +790,10 @@ class User < ActiveRecord::Base email.start_with?('temp-email-for-oauth') end - def avatar_url(size = nil, scale = 2) - if self[:avatar].present? - [gitlab_config.url, avatar.url].join - else - GravatarService.new.execute(email, size, scale) - end + def avatar_url(size: nil, scale: 2, **args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) || GravatarService.new.execute(email, size, scale) end def all_emails @@ -888,23 +917,36 @@ class User < ActiveRecord::Base @global_notification_setting end - def assigned_open_merge_request_count(force: false) - Rails.cache.fetch(['users', id, 'assigned_open_merge_request_count'], force: force) do - assigned_merge_requests.opened.count + def assigned_open_merge_requests_count(force: false) + Rails.cache.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force) do + MergeRequestsFinder.new(self, assignee_id: self.id, state: 'opened').execute.count end end def assigned_open_issues_count(force: false) Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force) do - assigned_issues.opened.count + IssuesFinder.new(self, assignee_id: self.id, state: 'opened').execute.count end end def update_cache_counts - assigned_open_merge_request_count(force: true) + assigned_open_merge_requests_count(force: true) assigned_open_issues_count(force: true) end + def invalidate_cache_counts + invalidate_issue_cache_counts + invalidate_merge_request_cache_counts + end + + def invalidate_issue_cache_counts + Rails.cache.delete(['users', id, 'assigned_open_issues_count']) + end + + def invalidate_merge_request_cache_counts + Rails.cache.delete(['users', id, 'assigned_open_merge_requests_count']) + end + def todos_done_count(force: false) Rails.cache.fetch(['users', id, 'todos_done_count'], force: force) do TodosFinder.new(self, state: :done).execute.count @@ -953,6 +995,15 @@ class User < ActiveRecord::Base self.admin = (new_level == 'admin') end + def update_two_factor_requirement + periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) + + self.require_two_factor_authentication_from_group = periods.any? + self.two_factor_grace_period = periods.min || User.column_defaults['two_factor_grace_period'] + + save + end + protected # override, from Devise::Validatable @@ -977,6 +1028,15 @@ class User < ActiveRecord::Base devise_mailer.send(notification, self, *args).deliver_later end + # This works around a bug in Devise 4.2.0 that erroneously causes a user to + # be considered active in MySQL specs due to a sub-second comparison + # issue. For more details, see: https://gitlab.com/gitlab-org/gitlab-ee/issues/2362#note_29004709 + def confirmation_period_valid? + return false if self.class.allow_unconfirmed_access_for == 0.days + + super + end + def ensure_external_user_rights return unless external? @@ -1059,11 +1119,13 @@ class User < ActiveRecord::Base User.find_by_email(s) end - scope.create( + user = scope.build( username: username, email: email, &creation_block ) + user.save(validate: false) + user ensure Gitlab::ExclusiveLease.cancel(lease_key, uuid) end |