diff options
Diffstat (limited to 'lib')
118 files changed, 2074 insertions, 347 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 07f529b01bb..5c98b0ad56c 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -34,11 +34,11 @@ module API repository = user_project.repository branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute - branches = ::Kaminari.paginate_array(branches) + branches = paginate(::Kaminari.paginate_array(branches)) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) present( - paginate(branches), + branches, with: Entities::Branch, current_user: current_user, project: user_project, diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 2cd0d93b205..0871ea8d21e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -86,6 +86,10 @@ module API expose :admin?, as: :is_admin end + class UserDetailsWithAdmin < UserWithAdmin + expose :highest_role + end + class UserStatus < Grape::Entity expose :emoji expose :message @@ -271,7 +275,7 @@ module API expose :printing_merge_request_link_enabled expose :merge_method expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { - options[:statistics] && Ability.allowed?(options[:current_user], :download_code, project) + options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } # rubocop: disable CodeReuse/ActiveRecord @@ -1554,8 +1558,6 @@ module API class Suggestion < Grape::Entity expose :id - expose :from_original_line - expose :to_original_line expose :from_line expose :to_line expose :appliable?, as: :appliable diff --git a/lib/api/groups.rb b/lib/api/groups.rb index cb0d6d96f29..9fcf476f537 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -20,8 +20,19 @@ module API optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' end + if Gitlab.ee? + params :optional_params_ee do + optional :membership_lock, type: Boolean, desc: 'Prevent adding new members to project membership within this group' + optional :ldap_cn, type: String, desc: 'LDAP Common Name' + optional :ldap_access, type: Integer, desc: 'A valid access level' + optional :shared_runners_minutes_limit, type: Integer, desc: '(admin-only) Pipeline minutes quota for this group' + all_or_none_of :ldap_cn, :ldap_access + end + end + params :optional_params do use :optional_params_ce + use :optional_params_ee if Gitlab.ee? end params :statistics_params do @@ -164,6 +175,10 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params + + if Gitlab.ee? + optional :file_template_project_id, type: Integer, desc: 'The ID of a project to use for custom templates in this group' + end end put ':id' do group = find_group!(params[:id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b8bd180bdc1..8a21d44b4bf 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -302,6 +302,12 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + def filter_by_title(items, title) + items.where(title: title) + end + # rubocop: enable CodeReuse/ActiveRecord + def filter_by_search(items, text) items.search(text) end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index fe78049af87..3fd824877ae 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -5,9 +5,11 @@ module API module InternalHelpers attr_reader :redirected_path - def wiki? - set_project unless defined?(@wiki) # rubocop:disable Gitlab/ModuleWithInstanceVariables - @wiki # rubocop:disable Gitlab/ModuleWithInstanceVariables + delegate :wiki?, to: :repo_type + + def repo_type + set_project unless defined?(@repo_type) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @repo_type # rubocop:disable Gitlab/ModuleWithInstanceVariables end def project @@ -67,10 +69,10 @@ module API # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project if params[:gl_repository] - @project, @wiki = Gitlab::GlRepository.parse(params[:gl_repository]) + @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) @redirected_path = nil else - @project, @wiki, @redirected_path = Gitlab::RepoPath.parse(params[:project]) + @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project]) end end # rubocop:enable Gitlab/ModuleWithInstanceVariables @@ -78,7 +80,7 @@ module API # Project id to pass between components that don't share/don't have # access to the same filesystem mounts def gl_repository - Gitlab::GlRepository.gl_repository(project, wiki?) + repo_type.identifier_for_subject(project) end def gl_project_path @@ -92,7 +94,7 @@ module API # Return the repository depending on whether we want the wiki or the # regular repository def repository - if wiki? + if repo_type.wiki? project.wiki.repository else project.repository diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb new file mode 100644 index 00000000000..f6762910b0c --- /dev/null +++ b/lib/api/helpers/issues_helpers.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + module Helpers + module IssuesHelpers + def self.update_params_at_least_one_of + [ + :assignee_id, + :assignee_ids, + :confidential, + :created_at, + :description, + :discussion_locked, + :due_date, + :labels, + :milestone_id, + :state_event, + :title + ] + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index e6a72b949f9..7b858dc2e72 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -31,11 +31,50 @@ module API optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" end + if Gitlab.ee? + params :optional_project_params_ee do + optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' + optional :approvals_before_merge, type: Integer, desc: 'How many approvers should approve merge request by default' + optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' + optional :mirror, type: Boolean, desc: 'Enables pull mirroring in a project' + optional :mirror_trigger_builds, type: Boolean, desc: 'Pull mirroring triggers builds' + end + end + params :optional_project_params do use :optional_project_params_ce + use :optional_project_params_ee if Gitlab.ee? end end end + + def self.update_params_at_least_one_of + [ + :jobs_enabled, + :resolve_outdated_diff_discussions, + :ci_config_path, + :container_registry_enabled, + :default_branch, + :description, + :issues_enabled, + :lfs_enabled, + :merge_requests_enabled, + :merge_method, + :name, + :only_allow_merge_if_all_discussions_are_resolved, + :only_allow_merge_if_pipeline_succeeds, + :path, + :printing_merge_request_link_enabled, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :tag_list, + :visibility, + :wiki_enabled, + :avatar + ] + end end end end diff --git a/lib/api/helpers/search_helpers.rb b/lib/api/helpers/search_helpers.rb index 47fb5a36327..0e052e0e273 100644 --- a/lib/api/helpers/search_helpers.rb +++ b/lib/api/helpers/search_helpers.rb @@ -5,17 +5,17 @@ module API module SearchHelpers def self.global_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + %w(projects issues merge_requests milestones snippet_titles snippet_blobs users) end def self.group_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones) + %w(projects issues merge_requests milestones users) end def self.project_search_scopes # This is a separate method so that EE can redefine it. - %w(issues merge_requests milestones notes wiki_blobs commits blobs) + %w(issues merge_requests milestones notes wiki_blobs commits blobs users) end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 7f4a00f1389..cb9aa849eeb 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -59,7 +59,7 @@ module API actor end - access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_checker_klass = repo_type.access_checker_class access_checker = access_checker_klass.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, namespace_path: namespace_path, project_path: project_path, diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b2ec4ed898e..fae20e45bf9 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -8,15 +8,6 @@ module API helpers ::Gitlab::IssuableMetadata - # EE::API::Issues would override the following helpers - helpers do - params :issues_params_ee do - end - - params :issue_params_ee do - end - end - helpers do # rubocop: disable CodeReuse/ActiveRecord def find_issues(args = {}) @@ -33,6 +24,16 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + if Gitlab.ee? + params :issues_params_ee do + optional :weight, types: [Integer, String], integer_none_any: true, desc: 'The weight of the issue' + end + + params :issue_params_ee do + optional :weight, type: Integer, desc: 'The weight of the issue' + end + end + params :issues_params do optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' @@ -57,7 +58,7 @@ module API optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' use :pagination - use :issues_params_ee + use :issues_params_ee if Gitlab.ee? end params :issue_params do @@ -70,7 +71,7 @@ module API optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - use :issue_params_ee + use :issue_params_ee if Gitlab.ee? end end @@ -219,8 +220,8 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, - :labels, :created_at, :due_date, :confidential, :state_event + + at_least_one_of(*Helpers::IssuesHelpers.update_params_at_least_one_of) end # rubocop: disable CodeReuse/ActiveRecord put ':id/issues/:issue_iid' do diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index a0ca39b69d4..62e159ab003 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -16,6 +16,7 @@ module API optional :state, type: String, values: %w[active closed all], default: 'all', desc: 'Return "active", "closed", or "all" milestones' optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' + optional :title, type: String, desc: 'The title of the milestones' optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' use :pagination end @@ -33,6 +34,7 @@ module API milestones = parent.milestones.order_id_desc milestones = Milestone.filter_by_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? + milestones = filter_by_title(milestones, params[:title]) if params[:title] milestones = filter_by_search(milestones, params[:search]) if params[:search] present paginate(milestones), with: Entities::Milestone diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 91501ba4d36..0f4a47677d9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,12 +11,20 @@ module API before { authenticate_non_get! } helpers do - params :optional_filter_params_ee do - # EE::API::Projects would override this helper - end + if Gitlab.ee? + params :optional_filter_params_ee do + optional :wiki_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where wiki checksum is failed' + optional :repository_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where repository checksum is failed' + end - params :optional_update_params_ee do - # EE::API::Projects would override this helper + params :optional_update_params_ee do + optional :mirror_user_id, type: Integer, desc: 'User responsible for all the activity surrounding a pull mirror event' + optional :only_mirror_protected_branches, type: Grape::API::Boolean, desc: 'Only mirror protected branches' + optional :mirror_overwrites_diverged_branches, type: Grape::API::Boolean, desc: 'Pull mirror overwrites diverged branches' + optional :import_url, type: String, desc: 'URL from which the project is imported' + optional :packages_enabled, type: Grape::API::Boolean, desc: 'Enable project packages feature' + optional :fallback_approvals_required, type: Integer, desc: 'Overall approvals required when no rule is present' + end end # EE::API::Projects would override this method @@ -35,34 +43,6 @@ module API end end - def self.update_params_at_least_one_of - [ - :jobs_enabled, - :resolve_outdated_diff_discussions, - :ci_config_path, - :container_registry_enabled, - :default_branch, - :description, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :merge_method, - :name, - :only_allow_merge_if_all_discussions_are_resolved, - :only_allow_merge_if_pipeline_succeeds, - :path, - :printing_merge_request_link_enabled, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :tag_list, - :visibility, - :wiki_enabled, - :avatar - ] - end - helpers do params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' @@ -97,7 +77,7 @@ module API optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' - use :optional_filter_params_ee + use :optional_filter_params_ee if Gitlab.ee? end params :create_params do @@ -316,8 +296,9 @@ module API optional :path, type: String, desc: 'The path of the repository' use :optional_project_params + use :optional_update_params_ee if Gitlab.ee? - at_least_one_of(*::API::Projects.update_params_at_least_one_of) + at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end put ':id' do authorize_admin_project diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 5af43448727..f8cce1ed784 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -51,6 +51,30 @@ module API optional :merge_access_level, type: Integer, values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' + + if Gitlab.ee? + optional :unprotect_access_level, type: Integer, + values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels, + desc: 'Access levels allowed to unprotect (defaults: `40`, maintainer access level)' + + optional :allowed_to_push, type: Array, desc: 'An array of users/groups allowed to push' do + optional :access_level, type: Integer, values: ProtectedBranch::PushAccessLevel.allowed_access_levels + optional :user_id, type: Integer + optional :group_id, type: Integer + end + + optional :allowed_to_merge, type: Array, desc: 'An array of users/groups allowed to merge' do + optional :access_level, type: Integer, values: ProtectedBranch::MergeAccessLevel.allowed_access_levels + optional :user_id, type: Integer + optional :group_id, type: Integer + end + + optional :allowed_to_unprotect, type: Array, desc: 'An array of users/groups allowed to unprotect' do + optional :access_level, type: Integer, values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels + optional :user_id, type: Integer + optional :group_id, type: Integer + end + end end # rubocop: disable CodeReuse/ActiveRecord post ':id/protected_branches' do diff --git a/lib/api/search.rb b/lib/api/search.rb index f65e810bf90..60095300ea1 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -17,7 +17,8 @@ module API blobs: Entities::Blob, wiki_blobs: Entities::Blob, snippet_titles: Entities::Snippet, - snippet_blobs: Entities::Snippet + snippet_blobs: Entities::Snippet, + users: Entities::UserBasic }.freeze def search(additional_params = {}) @@ -51,6 +52,12 @@ module API # Defining this method here as a noop allows us to easily extend it in # EE, without having to modify this file directly. end + + def check_users_search_allowed! + if params[:scope].to_sym == :users && Feature.disabled?(:users_search, default_enabled: true) + render_api_error!({ error: _("Scope not supported with disabled 'users_search' feature!") }, 400) + end + end end resource :search do @@ -67,6 +74,7 @@ module API end get do verify_search_scope! + check_users_search_allowed! present search, with: entity end @@ -87,6 +95,7 @@ module API end get ':id/(-/)search' do verify_search_scope! + check_users_search_allowed! present search(group_id: user_group.id), with: entity end @@ -106,6 +115,8 @@ module API use :pagination end get ':id/(-/)search' do + check_users_search_allowed! + present search(project_id: user_project.id), with: entity end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 3cb2f69c4ef..d742c6c97c1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -135,8 +135,44 @@ module API desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." end + if Gitlab.ee? + optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch' + + given elasticsearch_aws: ->(val) { val } do + optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key' + requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured' + optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key' + end + + optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing' + + given elasticsearch_indexing: ->(val) { val } do + optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search' + requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")' + end + + optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons' + optional :help_text, type: String, desc: 'GitLab server administrator information' + optional :repository_size_limit, type: Integer, desc: 'Size limit per repository (MB)' + optional :file_template_project_id, type: Integer, desc: 'ID of project where instance-level file templates are stored.' + optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.' + optional :snowplow_enabled, type: Boolean, desc: 'Enable Snowplow' + + given snowplow_enabled: ->(val) { val } do + requires :snowplow_collector_uri, type: String, desc: 'Snowplow Collector URI' + optional :snowplow_cookie_domain, type: String, desc: 'Snowplow cookie domain' + optional :snowplow_site_id, type: String, desc: 'Snowplow Site/Application ID' + end + + optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' + end + optional_attributes = ::ApplicationSettingsHelper.visible_attributes << :performance_bar_allowed_group_id + if Gitlab.ee? + optional_attributes += EE::ApplicationSettingsHelper.possible_licensed_attributes + end + optional(*optional_attributes) at_least_one_of(*optional_attributes) end diff --git a/lib/api/users.rb b/lib/api/users.rb index 7d88880d412..776329622e2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -51,6 +51,10 @@ module API optional :avatar, type: File, desc: 'Avatar image for user' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider + + if Gitlab.ee? + optional :shared_runners_minutes_limit, type: Integer, desc: 'Pipeline minutes quota for this user' + end end params :sort_params do @@ -80,6 +84,10 @@ module API use :sort_params use :pagination use :with_custom_attributes + + if Gitlab.ee? + optional :skip_ldap, type: Boolean, default: false, desc: 'Skip LDAP users' + end end # rubocop: disable CodeReuse/ActiveRecord get do @@ -124,7 +132,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) - opts = { with: current_user&.admin? ? Entities::UserWithAdmin : Entities::User, current_user: current_user } + opts = { with: current_user&.admin? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user } user, opts = with_custom_attributes(user, opts) present user, opts diff --git a/lib/api/variables.rb b/lib/api/variables.rb index d0d81ebc870..3489ba827e4 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -55,6 +55,10 @@ module API requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + + if Gitlab.ee? + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + end end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -76,6 +80,10 @@ module API optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + + if Gitlab.ee? + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + end end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 9577df2634a..5a20b6ae0a6 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -9,7 +9,7 @@ module Backup def initialize(progress) @progress = progress - super('uploads', Rails.root.join('public/uploads')) + super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads")) end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 4764f8e1e19..5f8aca104aa 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -181,9 +181,10 @@ module Banzai title = object_link_title(object, matches) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, parent, object, - link_content: !!link_content, - link_reference: link_reference) + data_attributes = data_attributes_for(link_content || match, parent, object, + link_content: !!link_content, + link_reference: link_reference) + data = data_attribute(data_attributes) url = if matches.names.include?("url") && matches[:url] @@ -206,13 +207,13 @@ module Banzai def data_attributes_for(text, parent, object, link_content: false, link_reference: false) object_parent_type = parent.is_a?(Group) ? :group : :project - data_attribute( + { original: text, link: link_content, link_reference: link_reference, object_parent_type => parent.id, object_sym => object.id - ) + } end def object_link_text_extras(object, matches) diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 7098767b583..f05902078dc 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -20,7 +20,9 @@ module Banzai end def object_link_title(object, matches) - object_link_commit_title(object, matches) || super + # The method will return `nil` if object is not a commit + # allowing for properly handling the extended MR Tooltip + object_link_commit_title(object, matches) end def object_link_text_extras(object, matches) @@ -53,6 +55,14 @@ module Banzai .includes(target_project: :namespace) end + def reference_class(object_sym, options = {}) + super(object_sym, tooltip: false) + end + + def data_attributes_for(text, parent, object, data = {}) + super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title) + end + private def object_link_commit_title(object, matches) diff --git a/lib/banzai/filter/output_safety.rb b/lib/banzai/filter/output_safety.rb new file mode 100644 index 00000000000..d4ebce5d9c9 --- /dev/null +++ b/lib/banzai/filter/output_safety.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module OutputSafety + def escape_once(html) + html.html_safe? ? html : ERB::Util.html_escape_once(html) + end + end + end +end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 42f9b3a689c..b3ce9200b49 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -12,6 +12,7 @@ module Banzai # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter include RequestStoreReferenceCache + include OutputSafety class << self attr_accessor :reference_type @@ -43,10 +44,6 @@ module Banzai end.join(' ') end - def escape_once(html) - html.html_safe? ? html : ERB::Util.html_escape_once(html) - end - def ignore_ancestor_query @ignore_ancestor_query ||= begin parents = %w(pre code a style) diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb index 9950db373d8..848aca10a20 100644 --- a/lib/banzai/filter/suggestion_filter.rb +++ b/lib/banzai/filter/suggestion_filter.rb @@ -6,11 +6,15 @@ module Banzai class SuggestionFilter < HTML::Pipeline::Filter # Class used for tagging elements that should be rendered TAG_CLASS = 'js-render-suggestion'.freeze + SUGGESTION_REGEX = Gitlab::Diff::SuggestionsParser::SUGGESTION_CONTEXT def call return doc unless suggestions_filter_enabled? doc.search('pre.suggestion > code').each do |node| + # TODO: Remove once multi-line suggestions FF get removed (#59178). + remove_multi_line_params(node.parent) + node.add_class(TAG_CLASS) end @@ -20,6 +24,20 @@ module Banzai def suggestions_filter_enabled? context[:suggestions_filter_enabled] end + + private + + def project + context[:project] + end + + def remove_multi_line_params(node) + return if Feature.enabled?(:multi_line_suggestions, project) + + if node[SyntaxHighlightFilter::LANG_PARAMS_ATTR]&.match?(SUGGESTION_REGEX) + node.remove_attribute(SyntaxHighlightFilter::LANG_PARAMS_ATTR) + end + end end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9ffde52b5f2..fe56f9a1e33 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -8,6 +8,11 @@ module Banzai # HTML Filter to highlight fenced code blocks # class SyntaxHighlightFilter < HTML::Pipeline::Filter + include OutputSafety + + PARAMS_DELIMITER = ':'.freeze + LANG_PARAMS_ATTR = 'data-lang-params'.freeze + def call doc.search('pre > code').each do |node| highlight_node(node) @@ -18,7 +23,7 @@ module Banzai def highlight_node(node) css_classes = +'code highlight js-syntax-highlight' - lang = node.attr('lang') + lang, lang_params = parse_lang_params(node.attr('lang')) retried = false if use_rouge?(lang) @@ -46,7 +51,10 @@ module Banzai retry end - highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" + lang="#{language}" + #{lang_params} + v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -54,6 +62,15 @@ module Banzai private + def parse_lang_params(language) + return unless language + + lang, params = language.split(PARAMS_DELIMITER, 2) + formatted_params = %(#{LANG_PARAMS_ATTR}="#{escape_once(params)}") if params + + [lang, formatted_params] + end + # Separate method so it can be instrumented. def lex(lexer, code) lexer.lex(code) diff --git a/lib/banzai/suggestions_parser.rb b/lib/banzai/suggestions_parser.rb index 09f36635020..0d7f751bfc1 100644 --- a/lib/banzai/suggestions_parser.rb +++ b/lib/banzai/suggestions_parser.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# TODO: Delete when https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26107 +# exchange this parser by `Gitlab::Diff::SuggestionsParser`. module Banzai module SuggestionsParser # Returns the content of each suggestion code block. diff --git a/lib/gitlab.rb b/lib/gitlab.rb index f42ca5a9cd6..1204e53ee2e 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -24,7 +24,7 @@ module Gitlab if File.exist?(root.join("REVISION")) File.read(root.join("REVISION")).strip.freeze else - result = Gitlab::Popen.popen_with_detail(%W[#{config.git.bin_path} log --pretty=format:%h -n 1]) + result = Gitlab::Popen.popen_with_detail(%W[#{config.git.bin_path} log --pretty=format:%h --abbrev=11 -n 1]) if result.status.success? result.stdout.chomp.freeze diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index ec090aea784..6c8ca8f219c 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -46,6 +46,12 @@ module Gitlab ) end + def options_with_none + options_with_owner.merge( + "None" => NO_ACCESS + ) + end + def sym_options { guest: GUEST, @@ -75,12 +81,20 @@ module Gitlab def human_access(access) options_with_owner.key(access) end + + def human_access_with_none(access) + options_with_none.key(access) + end end def human_access Gitlab::Access.human_access(access_field) end + def human_access_with_none + Gitlab::Access.human_access_with_none(access_field) + end + def owner? access_field == OWNER end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 13d67e0f871..c1517222956 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -69,7 +69,7 @@ module Gitlab end def name - attribute_value(:name).first + attribute_value(:name)&.first end def uid diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index f38c5d57c44..09d1d79fefc 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -146,7 +146,6 @@ module Gitlab Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) - rescue Gitlab::Auth::LDAP::LDAPConnectionError nil end @@ -200,22 +199,19 @@ module Gitlab # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? username = ldap_person.username.presence + name = ldap_person.name.presence email = ldap_person.email.first.presence end username ||= auth_hash.username + name ||= auth_hash.name email ||= auth_hash.email valid_username = ::Namespace.clean_path(username) - - uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } - - name = auth_hash.name - name = valid_username if name.strip.empty? + valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } { - name: name, + name: name.strip.presence || valid_username, username: valid_username, email: email, password: auth_hash.password, @@ -248,8 +244,9 @@ module Gitlab metadata.provider = auth_hash.provider end - if creating_linked_ldap_user? && gl_user.email == ldap_person.email.first - metadata.set_attribute_synced(:email, true) + if creating_linked_ldap_user? + metadata.set_attribute_synced(:name, true) if gl_user.name == ldap_person.name + metadata.set_attribute_synced(:email, true) if gl_user.email == ldap_person.email.first metadata.provider = ldap_person.provider end end diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb new file mode 100644 index 00000000000..3fe72f5fd43 --- /dev/null +++ b/lib/gitlab/authorized_keys.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Gitlab + class AuthorizedKeys + KeyError = Class.new(StandardError) + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + def initialize(logger = Gitlab::AppLogger) + @logger = logger + end + + # Add id and its key to the authorized_keys file + # + # @param [String] id identifier of key prefixed by `key-` + # @param [String] key public key to be added + # @return [Boolean] + def add_key(id, key) + lock do + public_key = strip(key) + logger.info("Adding key (#{id}): #{public_key}") + open_authorized_keys_file('a') { |file| file.puts(key_line(id, public_key)) } + end + + true + end + + # Atomically add all the keys to the authorized_keys file + # + # @param [Array<::Key>] keys list of Key objects to be added + # @return [Boolean] + def batch_add_keys(keys) + lock(300) do # Allow 300 seconds (5 minutes) for batch_add_keys + open_authorized_keys_file('a') do |file| + keys.each do |key| + public_key = strip(key.key) + logger.info("Adding key (#{key.shell_id}): #{public_key}") + file.puts(key_line(key.shell_id, public_key)) + end + end + end + + true + rescue Gitlab::AuthorizedKeys::KeyError + false + end + + # Remove key by ID from the authorized_keys file + # + # @param [String] id identifier of the key to be removed prefixed by `key-` + # @return [Boolean] + def rm_key(id) + lock do + logger.info("Removing key (#{id})") + open_authorized_keys_file('r+') do |f| + while line = f.gets + next unless line.start_with?("command=\"#{command(id)}\"") + + f.seek(-line.length, IO::SEEK_CUR) + # Overwrite the line with #'s. Because the 'line' variable contains + # a terminating '\n', we write line.length - 1 '#' characters. + f.write('#' * (line.length - 1)) + end + end + end + + true + rescue Errno::ENOENT + false + end + + # Clear the authorized_keys file + # + # @return [Boolean] + def clear + open_authorized_keys_file('w') { |file| file.puts '# Managed by gitlab-rails' } + + true + end + + # Read the authorized_keys file and return IDs of each key + # + # @return [Array<Integer>] + def list_key_ids + logger.info('Listing all key IDs') + + [].tap do |a| + open_authorized_keys_file('r') do |f| + f.each_line do |line| + key_id = line.match(/key-(\d+)/) + + next unless key_id + + a << key_id[1].chomp.to_i + end + end + end + rescue Errno::ENOENT + [] + end + + private + + def lock(timeout = 10) + File.open("#{authorized_keys_file}.lock", "w+") do |f| + f.flock File::LOCK_EX + Timeout.timeout(timeout) { yield } + ensure + f.flock File::LOCK_UN + end + end + + def open_authorized_keys_file(mode) + File.open(authorized_keys_file, mode, 0o600) do |file| + file.chmod(0o600) + yield file + end + end + + def key_line(id, key) + key = key.chomp + + if key.include?("\n") || key.include?("\t") + raise KeyError, "Invalid public_key: #{key.inspect}" + end + + %Q(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) + end + + def command(id) + unless /\A[a-z0-9-]+\z/ =~ id + raise KeyError, "Invalid ID: #{id.inspect}" + end + + "#{File.join(Gitlab.config.gitlab_shell.path, 'bin', 'gitlab-shell')} #{id}" + end + + def strip(key) + key.split(/[ ]+/)[0, 2].join(' ') + end + + def authorized_keys_file + Gitlab.config.gitlab_shell.authorized_keys_file + end + end +end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb new file mode 100644 index 00000000000..a4c6540c61b --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration creates records on merge_request_assignees according + # to the given merge request IDs range. A _single_ INSERT is issued for the given range. + # This is required for supporting multiple assignees on merge requests. + class PopulateMergeRequestAssigneesTable + def perform(from_id, to_id) + select_sql = + MergeRequest + .where(merge_request_assignees_not_exists_clause) + .where(id: from_id..to_id) + .where('assignee_id IS NOT NULL') + .select(:id, :assignee_id) + .to_sql + + execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") + end + + private + + def merge_request_assignees_not_exists_clause + <<~SQL + NOT EXISTS (SELECT 1 FROM merge_request_assignees + WHERE merge_request_assignees.merge_request_id = merge_requests.id) + SQL + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb index 64c3dfcd10b..2c5f9654496 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -15,6 +15,7 @@ module Gitlab failed: '#e05d44', running: '#dfb317', pending: '#dfb317', + preparing: '#dfb317', canceled: '#9f9f9f', skipped: '#9f9f9f', unknown: '#9f9f9f' diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index ad926739752..1dbd564fb6f 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -59,6 +59,8 @@ module Gitlab def protected_branch_creation_checks logger.log_timed(LOG_MESSAGES[:protected_branch_creation_checks]) do + break if user_access.can_push_to_branch?(branch_name) + unless user_access.can_merge_to_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] end diff --git a/lib/gitlab/ci/build/prerequisite/base.rb b/lib/gitlab/ci/build/prerequisite/base.rb new file mode 100644 index 00000000000..156aa22d95b --- /dev/null +++ b/lib/gitlab/ci/build/prerequisite/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Prerequisite + class Base + include Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def unmet? + raise NotImplementedError + end + + def complete! + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/prerequisite/factory.rb b/lib/gitlab/ci/build/prerequisite/factory.rb new file mode 100644 index 00000000000..60cdf7af418 --- /dev/null +++ b/lib/gitlab/ci/build/prerequisite/factory.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Prerequisite + class Factory + attr_reader :build + + def self.prerequisites + [KubernetesNamespace] + end + + def initialize(build) + @build = build + end + + def unmet + build_prerequisites.select(&:unmet?) + end + + private + + def build_prerequisites + self.class.prerequisites.map do |prerequisite| + prerequisite.new(build) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb new file mode 100644 index 00000000000..41135ae62bb --- /dev/null +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Prerequisite + class KubernetesNamespace < Base + def unmet? + deployment_cluster.present? && kubernetes_namespace.new_record? + end + + def complete! + return unless unmet? + + create_or_update_namespace + end + + private + + def deployment_cluster + build.deployment&.cluster + end + + def kubernetes_namespace + strong_memoize(:kubernetes_namespace) do + deployment_cluster.find_or_initialize_kubernetes_namespace_for_project(build.project) + end + end + + def create_or_update_namespace + Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService.new( + cluster: deployment_cluster, + kubernetes_namespace: kubernetes_namespace + ).execute + end + end + end + end + end +end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb index fbdb84c0522..1625cb841b6 100644 --- a/lib/gitlab/ci/model.rb +++ b/lib/gitlab/ci/model.rb @@ -8,7 +8,7 @@ module Gitlab end def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize) end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 7b77e86feae..03af99ba9a5 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data + :chat_data, :allow_mirror_update ) do include Gitlab::Utils::StrongMemoize @@ -33,6 +33,13 @@ module Gitlab end end + def merge_request_ref_exists? + strong_memoize(:merge_request_ref_exists) do + MergeRequest.merge_request_ref?(origin_ref) && + project.repository.ref_exists?(origin_ref) + end + end + def ref strong_memoize(:ref) do Gitlab::Git.ref_name(origin_ref) diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index ebd7e6e8289..aaa3daddcc5 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -44,6 +44,8 @@ module Gitlab access.can_update_branch?(@command.ref) elsif @command.tag_exists? access.can_create_tag?(@command.ref) + elsif @command.merge_request_ref_exists? + access.can_update_branch?(@command.merge_request.source_branch) else true # Allow it for now and we'll reject when we check ref existence end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 9c6c2bc8e25..8f5445850d7 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -9,7 +9,7 @@ module Gitlab include Chain::Helpers def perform! - unless @command.branch_exists? || @command.tag_exists? + unless @command.branch_exists? || @command.tag_exists? || @command.merge_request_ref_exists? return error('Reference not found') end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 6e4bfe23f2b..f7d0715e617 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -11,6 +11,7 @@ module Gitlab Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, + Status::Build::Preparing, Status::Build::Pending, Status::Build::Skipped], [Status::Build::Cancelable, diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index d40454df737..76dfe7b7639 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -15,7 +15,8 @@ module Gitlab runner_unsupported: 'unsupported runner', stale_schedule: 'stale schedule', job_execution_timeout: 'job execution timeout', - archived_failure: 'archived failure' + archived_failure: 'archived failure', + unmet_prerequisites: 'unmet prerequisites' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/build/preparing.rb b/lib/gitlab/ci/status/build/preparing.rb new file mode 100644 index 00000000000..1fddcb05f79 --- /dev/null +++ b/lib/gitlab/ci/status/build/preparing.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class Preparing < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/job_not_triggered.svg', + size: 'svg-306', + title: _('This job is preparing to start'), + content: _('This job is performing tasks that must complete before it can start') + } + end + + def self.matches?(build, _) + build.preparing? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb new file mode 100644 index 00000000000..62985d0a9f9 --- /dev/null +++ b/lib/gitlab/ci/status/preparing.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class Preparing < Status::Core + def text + s_('CiStatusText|preparing') + end + + def label + s_('CiStatusLabel|preparing') + end + + ## + # TODO: shared with 'created' + # until we get one for 'preparing' + # + def icon + 'status_created' + end + + ## + # TODO: shared with 'created' + # until we get one for 'preparing' + # + def favicon + 'favicon_status_created' + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 9c534b2b8e7..8e767b22360 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -118,4 +118,3 @@ promoteProduction: - master script: - bundle exec fastlane promote_beta_to_production -
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cbe466a1c37..78872b3bbe3 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -697,6 +697,8 @@ rollout 100%: helm upgrade --install \ --wait \ --set service.enabled="$service_enabled" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ --set image.repository="$CI_APPLICATION_REPOSITORY" \ --set image.tag="$CI_APPLICATION_TAG" \ @@ -734,6 +736,8 @@ rollout 100%: helm upgrade --install \ --wait \ --set service.enabled="$service_enabled" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ --set image.repository="$CI_APPLICATION_REPOSITORY" \ --set image.tag="$CI_APPLICATION_TAG" \ @@ -852,7 +856,7 @@ rollout 100%: function check_kube_domain() { ensure_kube_ingress_base_domain - if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then + if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then echo "In order to deploy or use Review Apps," echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 2d218b2e164..368069844ea 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -7,28 +7,28 @@ before_script: - echo "Before script section" - echo "For example you might run an update here or install a build dependency" - echo "Or perhaps you might print out some debugging details" - + after_script: - echo "After script section" - echo "For example you might do some cleanup here" - + build1: stage: build script: - echo "Do your build here" - + test1: stage: test - script: + script: - echo "Do a test here" - echo "For example run a test suite" - + test2: stage: test - script: + script: - echo "Do another parallel test here" - echo "For example run a lint test" - + deploy1: stage: deploy script: diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index c83c49d8c95..9a8fa9d7091 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -7,9 +7,9 @@ build: stage: build # instead of calling g++ directly you can also use some build toolkit like make # install the necessary build tools when needed - # before_script: - # - apt update && apt -y install make autoconf - script: + # before_script: + # - apt update && apt -y install make autoconf + script: - g++ helloworld.cpp -o mybinary artifacts: paths: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 4d5b6484d6e..33507aa58e4 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -41,7 +41,7 @@ chefspec: # - apt-get -y install rsync # script: # - kitchen verify default-centos-6 --destroy=always -# +# #verify-centos-7: # stage: functional # before_script: diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index f066285b1ad..0610cb9ccc0 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -5,9 +5,9 @@ image: clojure:lein-2.7.0 # Make sure you configure the connection as well before_script: - # If you need to install any external applications, like a + # If you need to install any external applications, like a # postgres client, you may want to uncomment the line below: - # + # #- apt-get update -y # # Retrieve project dependencies @@ -17,6 +17,6 @@ before_script: test: script: - # If you need to run any migrations or configure the database, this - # would be the point to do it. + # If you need to run any migrations or configure the database, this + # would be the point to do it. - lein test diff --git a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml new file mode 100644 index 00000000000..10b25af904f --- /dev/null +++ b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml @@ -0,0 +1,17 @@ +code_quality: + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + variables: + DOCKER_DRIVER: overlay2 + script: + - docker run + --env SOURCE_CODE="$PWD" + --volume "$PWD":/code + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/codequality:11-8-stable" /code + artifacts: + reports: + codequality: gl-code-quality-report.json + expire_in: 1 week diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index 57afcbbe8b5..1d8be6f017e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -21,7 +21,7 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging # Uncomment next line if your Django app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - pip install -r requirements.txt diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 48d98dddfad..cbf4d58bdad 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -23,7 +23,6 @@ build: - build - .gradle - test: stage: test script: gradle check @@ -33,4 +32,3 @@ test: paths: - build - .gradle - diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 7fc698d50cf..dbc868238f8 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -13,7 +13,7 @@ image: java:8 variables: GRAILS_VERSION: "3.1.9" GRADLE_VERSION: "2.13" - + # We use SDKMan as tool for managing versions before_script: - apt-get update -qq && apt-get install -y -qq unzip @@ -23,10 +23,10 @@ before_script: - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION # As it's not a good idea to version gradle.properties feel free to add your -# environments variable here +# environments variable here - echo grailsVersion=$GRAILS_VERSION > gradle.properties - echo gradleWrapperVersion=2.14 >> gradle.properties -# refresh dependencies from your project +# refresh dependencies from your project - ./gradlew --refresh-dependencies # Be aware that if you are using Angular profile, # Bower cannot be run as root if you don't allow it before. @@ -36,5 +36,5 @@ before_script: # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). build: - script: - - ./gradlew build
\ No newline at end of file + script: + - ./gradlew build diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 04c21b4725d..2c4683fbfbb 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -30,7 +30,7 @@ test:0.7: image: julia:0.7 <<: *test_definition - + test:1.0: image: julia:1.0 <<: *test_definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index d0cad285572..e1cd29ecc94 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -22,33 +22,25 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - # Update packages + # Update packages - apt-get update -yqq - # Prep for Node - apt-get install gnupg -yqq - # Upgrade to Node 8 - curl -sL https://deb.nodesource.com/setup_8.x | bash - - # Install dependencies - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq - # Install php extensions - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache - # Install & enable Xdebug for code coverage reports - pecl install xdebug - docker-php-ext-enable xdebug - # Install Composer and project dependencies. - curl -sS https://getcomposer.org/installer | php - php composer.phar install - # Install Node dependencies. # comment this out if you don't have a node dependency - npm install - # Copy over testing configuration. # Don't forget to set the database config in .env.testing correctly # DB_HOST=mysql @@ -56,20 +48,16 @@ before_script: # DB_USERNAME=root # DB_PASSWORD=secret - cp .env.testing .env - # Run npm build # comment this out if you don't have a frontend build # you can change this to to your frontend building script like # npm run build - npm run dev - # Generate an application key. Re-cache. - php artisan key:generate - php artisan config:cache - # Run database migrations. - php artisan migrate - # Run database seed - php artisan db:seed @@ -77,7 +65,6 @@ test: script: # run laravel tests - php vendor/bin/phpunit --coverage-text --colors=never - # run frontend tests # if you have any task for testing frontend # set it in your package.json script diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 492b3d03db2..c9838c7a7ff 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -66,7 +66,6 @@ verify:jdk8: <<: *verify image: maven:3.3.9-jdk-8 - # For `master` branch run `mvn deploy` automatically. # Here you need to decide whether you want to use JDK7 or 8. # To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. @@ -85,7 +84,6 @@ deploy:jdk8: - target/staging image: maven:3.3.9-jdk-8 - pages: image: busybox:latest stage: deploy diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 3585f99760f..86d62b93313 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -32,11 +32,11 @@ release: # The output path is relative to the position of the csproj-file - msbuild /p:Configuration="Release" /p:Platform="Any CPU" /p:OutputPath="./../../build/release/" "MyProject.sln" - + debug: stage: test script: # The output path is relative to the position of the csproj-file - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" /p:OutputPath="./../../build/debug/" "MyProject.sln" - - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 7fcc0b436b5..d6de8cab5d1 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules/ - script: - npm install -g brunch - brunch build --production diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index dd3ef149668..4b58003ee10 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules - script: - npm install -g harp - harp compile ./ public diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index b8cfb0f56f6..f9ddcc6fb0a 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -9,7 +9,7 @@ pages: - public only: - master - + test: script: - hugo diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 7abfaf53e8e..7a485f8d135 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -13,7 +13,6 @@ image: java:8 variables: JBAKE_VERSION: 2.5.1 - # We use SDKMan as tool for managing versions before_script: - apt-get update -qq && apt-get install -y -qq unzip zip @@ -29,4 +28,4 @@ pages: - jbake . public artifacts: paths: - - public
\ No newline at end of file + - public diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 0e5fb410a4e..5ca4619e200 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -11,24 +11,19 @@ cache: - node_modules/ before_script: - # Update packages + # Update packages - apt-get update -yqq - # Install dependencies - apt-get install -yqq gnupg zlib1g-dev libpng-dev - # Install Node 8 - curl -sL https://deb.nodesource.com/setup_8.x | bash - - apt-get install -yqq nodejs - # Install php extensions - docker-php-ext-install zip - - # Install Composer and project dependencies. + # Install Composer and project dependencies - curl -sS https://getcomposer.org/installer | php - - php composer.phar install - - # Install Node dependencies. + - php composer.phar install + # Install Node dependencies - npm install pages: diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index 50e8b7ccd46..c6ded272150 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules/ - script: - npm install -g metalsmith - npm install diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 098abe4daf5..3eaed4e91cd 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -18,7 +18,7 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate @@ -26,7 +26,7 @@ before_script: test: script: - python setup.py test - - pip install tox flake8 # you can also use tox + - pip install tox flake8 # you can also use tox - tox -e py36,flake8 run: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 0d12cbc6460..93196dbd475 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -21,7 +21,7 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - ruby -v # Print out ruby version for debugging + - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 4e708f229cd..ef6d7866e85 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -21,20 +21,19 @@ dast: allow_failure: true services: - docker:stable-dind - before_script: + script: - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | function dast_run() { docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" + --env DAST_TARGET_AVAILABILITY_TIMEOUT \ + --volume "$PWD:/output" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -w /output \ + "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ + /analyze -t $DAST_WEBSITE \ + "$@" } - script: - | if [ -n "$DAST_AUTH_URL" ] then diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 25a32ba0f74..5e128b793d0 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -26,7 +26,6 @@ variables: MSI_RELEASE_FOLDER: 'Setup\bin\Release' TEST_FOLDER: 'Tests\bin\Release' DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' - NUGET_PATH: 'C:\NuGet\nuget.exe' MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' @@ -84,4 +83,3 @@ deploy_job: dependencies: - build_job - test_job -
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 245e6bec60a..df6ac4d340d 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -17,7 +17,7 @@ variables: LC_ALL: "en_US.UTF-8" LANG: "en_US.UTF-8" GIT_STRATEGY: clone - + build: stage: build script: diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 552aad83dd4..469a7fd9f7b 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -47,7 +47,7 @@ module Gitlab # defaults for missing columns. if ActiveRecord::Migrator.needs_migration? db_attributes = current_settings&.attributes || {} - ::ApplicationSetting.build_from_defaults(db_attributes) + fake_application_settings(db_attributes) elsif current_settings.present? current_settings else diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d2b7ca015d4..ac65cf74808 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -108,7 +108,23 @@ module Gitlab %r{\A(ee/)?public/} => :frontend, %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, %r{\A(ee/)?vendor/assets/} => :frontend, - %r{\A(jest\.config\.js|package\.json|yarn\.lock)\z} => :frontend, + %r{\Ascripts/frontend/} => :frontend, + %r{(\A|/)( + \.babelrc | + \.eslintignore | + \.eslintrc(\.yml)? | + \.nvmrc | + \.prettierignore | + \.prettierrc | + \.scss-lint.yml | + \.stylelintrc | + babel\.config\.js | + jest\.config\.js | + karma\.config\.js | + webpack\.config\.js | + package\.json | + yarn\.lock + )\z}x => :frontend, %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e2637ad602a..8da98cc3909 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -27,6 +27,10 @@ module Gitlab config['adapter'] end + def self.human_adapter_name + postgresql? ? 'PostgreSQL' : 'MySQL' + end + def self.mysql? adapter_name.casecmp('mysql2').zero? end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index dbee47a19ee..c46087e65de 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -133,11 +133,15 @@ module Gitlab end def new_blob - new_blob_lazy&.itself + strong_memoize(:new_blob) do + new_blob_lazy&.itself + end end def old_blob - old_blob_lazy&.itself + strong_memoize(:old_blob) do + old_blob_lazy&.itself + end end def new_blob_lines_between(from_line, to_line) @@ -158,7 +162,10 @@ module Gitlab new_blob || old_blob end - attr_writer :highlighted_diff_lines + def highlighted_diff_lines=(value) + clear_memoization(:diff_lines_for_serializer) + @highlighted_diff_lines = value + end # Array of Gitlab::Diff::Line objects def diff_lines @@ -314,19 +321,21 @@ module Gitlab # This adds the bottom match line to the array if needed. It contains # the data to load more context lines. def diff_lines_for_serializer - lines = highlighted_diff_lines + strong_memoize(:diff_lines_for_serializer) do + lines = highlighted_diff_lines - return if lines.empty? - return if blob.nil? + next if lines.empty? + next if blob.nil? - last_line = lines.last + last_line = lines.last - if last_line.new_pos < total_blob_lines(blob) && !deleted_file? - match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) - lines.push(match_line) - end + if last_line.new_pos < total_blob_lines(blob) && !deleted_file? + match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) + lines.push(match_line) + end - lines + lines + end end def fully_expanded? diff --git a/lib/gitlab/diff/suggestion.rb b/lib/gitlab/diff/suggestion.rb new file mode 100644 index 00000000000..027c7a31bcf --- /dev/null +++ b/lib/gitlab/diff/suggestion.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class Suggestion + include Suggestible + include Gitlab::Utils::StrongMemoize + + attr_reader :diff_file, :lines_above, :lines_below, + :target_line + + def initialize(text, line:, above:, below:, diff_file:) + @text = text + @target_line = line + @lines_above = above.to_i + @lines_below = below.to_i + @diff_file = diff_file + end + + def to_hash + { + from_content: from_content, + to_content: to_content, + lines_above: @lines_above, + lines_below: @lines_below + } + end + + def from_content + strong_memoize(:from_content) do + fetch_from_content + end + end + + def to_content + # The parsed suggestion doesn't have information about the correct + # ending characters (we may have a line break, or not), so we take + # this information from the last line being changed (last + # characters). + endline_chars = line_break_chars(from_content.lines.last) + "#{@text}#{endline_chars}" + end + + private + + def line_break_chars(line) + match = /\r\n|\r|\n/.match(line) + match[0] if match + end + end + end +end diff --git a/lib/gitlab/diff/suggestion_diff.rb b/lib/gitlab/diff/suggestion_diff.rb new file mode 100644 index 00000000000..ee153c226b7 --- /dev/null +++ b/lib/gitlab/diff/suggestion_diff.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class SuggestionDiff + include Gitlab::Utils::StrongMemoize + + delegate :from_content, :to_content, :from_line, to: :@suggestible + + def initialize(suggestible) + @suggestible = suggestible + end + + def diff_lines + Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + end + + private + + def raw_diff + "#{diff_header}\n#{from_content_as_diff}#{to_content_as_diff}" + end + + def diff_header + "@@ -#{from_line} +#{from_line}" + end + + def from_content_as_diff + from_content.lines.map { |line| line.prepend('-') }.join + end + + def to_content_as_diff + to_content.lines.map { |line| line.prepend('+') }.join + end + end + end +end diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb new file mode 100644 index 00000000000..c8c03d5d001 --- /dev/null +++ b/lib/gitlab/diff/suggestions_parser.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class SuggestionsParser + # Matches for instance "-1", "+1" or "-1+2". + SUGGESTION_CONTEXT = /^(\-(?<above>\d+))?(\+(?<below>\d+))?$/.freeze + + class << self + # Returns an array of Gitlab::Diff::Suggestion which represents each + # suggestion in the given text. + # + def parse(text, position:, project:) + return [] unless position.complete? + + html = Banzai.render(text, project: nil, no_original_data: true) + doc = Nokogiri::HTML(html) + suggestion_nodes = doc.search('pre.suggestion') + + return [] if suggestion_nodes.empty? + + diff_file = position.diff_file(project.repository) + + suggestion_nodes.map do |node| + lang_param = node['data-lang-params'] + + lines_above, lines_below = nil + + if lang_param && suggestion_params = fetch_suggestion_params(lang_param) + lines_above, lines_below = + suggestion_params[:above], + suggestion_params[:below] + end + + Gitlab::Diff::Suggestion.new(node.text, + line: position.new_line, + above: lines_above.to_i, + below: lines_below.to_i, + diff_file: diff_file) + end + end + + private + + def fetch_suggestion_params(lang_param) + lang_param.match(SUGGESTION_CONTEXT) + end + end + end + end +end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bd806269bf0..77f7d9490f3 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -7,6 +7,8 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct + include ApplicationSettingImplementation + # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -26,20 +28,7 @@ module Gitlab FakeApplicationSettings.define_predicate_methods(options) end - def key_restriction_for(type) - 0 - end - - def allowed_key_types - ApplicationSetting::SUPPORTED_KEY_TYPES - end - - def pick_repository_storage - repository_storages.sample - end - - def commit_email_hostname - super.presence || ApplicationSetting.default_commit_email_hostname - end + alias_method :read_attribute, :[] + alias_method :has_attribute?, :[] end end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 1ae2f9dfd93..6e31064f737 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -10,7 +10,7 @@ module Gitlab elsif Gitlab::Utils.to_boolean(ENV['CANARY']) 'favicon-yellow.png' elsif Rails.env.development? - 'favicon-blue.png' + development_favicon else 'favicon.png' end @@ -18,6 +18,12 @@ module Gitlab ActionController::Base.helpers.image_path(image_name, host: host) end + def development_favicon + # This is a separate method so that EE can return a different favicon + # for development environments. + 'favicon-blue.png' + end + def status_overlay(status_name) path = File.join( 'ci_favicons', diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index e5bbd500e98..8fac3621df9 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -184,11 +184,12 @@ module Gitlab end end - def initialize(repository, raw_commit, head = nil) + def initialize(repository, raw_commit, head = nil, lazy_load_parents: false) raise "Nil as raw commit passed" unless raw_commit @repository = repository @head = head + @lazy_load_parents = lazy_load_parents init_commit(raw_commit) end @@ -225,6 +226,12 @@ module Gitlab author_name != committer_name || author_email != committer_email end + def parent_ids + return @parent_ids unless @lazy_load_parents + + @parent_ids ||= @repository.commit(id).parent_ids + end + def parent_id parent_ids.first end @@ -311,6 +318,10 @@ module Gitlab parent_ids.size > 1 end + def gitaly_commit? + raw_commit.is_a?(Gitaly::GitCommit) + end + def tree_entry(path) return unless path.present? @@ -333,7 +344,7 @@ module Gitlab end def to_gitaly_commit - return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) + return raw_commit if gitaly_commit? message_split = raw_commit.message.split("\n", 2) Gitaly::GitCommit.new( diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index 03caace6fce..b46d4ba0b02 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -4,19 +4,38 @@ module Gitlab module Git # # PreReceiveError is special because its message gets displayed to users - # in the web UI. To prevent XSS we sanitize the message on - # initialization. + # in the web UI. Because of this, we: + # - Only display errors that have been marked as safe with a prefix. + # This is to prevent leaking of stacktraces, or other sensitive info. + # - Sanitize the string of any XSS class PreReceiveError < StandardError - def initialize(msg = '') - super(nlbr(msg)) + SAFE_MESSAGE_PREFIXES = [ + 'GitLab:', # Messages from gitlab-shell + 'GL-HOOK-ERR:' # Messages marked as safe by user + ].freeze + + SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/ + + def initialize(message = '') + super(sanitize(message)) end private # In gitaly-ruby we override this method to do nothing, so that # sanitization happens in gitlab-rails only. - def nlbr(str) - Gitlab::Utils.nlbr(str) + def sanitize(message) + return message if message.blank? + + safe_messages = message.split("\n").map do |msg| + if (match = msg.match(SAFE_MESSAGE_REGEX)) + match[:safe_message].presence + end + end + + safe_messages = safe_messages.compact.join("\n") + + Gitlab::Utils.nlbr(safe_messages) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 35dd042ba6a..7d6851a4b8d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -344,12 +344,12 @@ module Gitlab end end - def new_blobs(newrev) + def new_blobs(newrev, dynamic_timeout: nil) return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA strong_memoize("new_blobs_#{newrev}") do wrapped_gitaly_errors do - gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT) + gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e3b9a7a1a89..f240387c28c 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,12 +33,6 @@ module Gitlab MUTEX = Mutex.new - class << self - attr_accessor :query_time - end - - self.query_time = 0 - define_histogram :gitaly_controller_action_duration_seconds do docstring "Gitaly endpoint histogram by controller and action combination" base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) @@ -174,6 +168,18 @@ module Gitlab add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) end + def self.query_time + SafeRequestStore[:gitaly_query_time] ||= 0 + end + + def self.query_time=(duration) + SafeRequestStore[:gitaly_query_time] = duration + end + + def self.query_time_ms + (self.query_time * 1000).round(2) + end + def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end @@ -296,6 +302,26 @@ module Gitlab end end + # Normally a FindCommit RPC will cache the commit with its SHA + # instead of a ref name, since it's possible the branch is mutated + # afterwards. However, for read-only requests that never mutate the + # branch, this method allows caching of the ref name directly. + def self.allow_ref_name_caching + return yield unless Gitlab::SafeRequestStore.active? + return yield if ref_name_caching_allowed? + + begin + Gitlab::SafeRequestStore[:allow_ref_name_caching] = true + yield + ensure + Gitlab::SafeRequestStore[:allow_ref_name_caching] = false + end + end + + def self.ref_name_caching_allowed? + Gitlab::SafeRequestStore[:allow_ref_name_caching] + end + def self.get_call_count(key) Gitlab::SafeRequestStore[key] || 0 end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ea12424eb4a..0d5debfcd01 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -286,7 +286,7 @@ module Gitlab commit = call_find_commit(revision) return unless commit - key[:commit_id] = commit.id + key[:commit_id] = commit.id unless GitalyClient.ref_name_caching_allowed? Gitlab::SafeRequestStore[key] = commit else call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index d5633d167ac..6f6698607d9 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -84,15 +84,22 @@ module Gitlab commits end - def list_new_blobs(newrev, limit = 0) + def list_new_blobs(newrev, limit = 0, dynamic_timeout: nil) request = Gitaly::ListNewBlobsRequest.new( repository: @gitaly_repo, commit_id: newrev, limit: limit ) + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + response = GitalyClient - .call(@storage, :ref_service, :list_new_blobs, request, timeout: GitalyClient.medium_timeout) + .call(@storage, :ref_service, :list_new_blobs, request, timeout: timeout) response.flat_map do |msg| # Returns an Array of Gitaly::NewBlobObject objects diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index e294173f992..1b293ddc7c7 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -89,7 +89,7 @@ module Gitlab return if project.repository.branch_exists?(source_branch) - project.repository.add_branch(merge_request.author, source_branch, pull_request.source_branch_sha) + project.repository.add_branch(project.creator, source_branch, pull_request.source_branch_sha) rescue Gitlab::Git::CommandError => e Gitlab::Sentry.track_acceptable_exception(e, extra: { diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index e2dfb00dcc5..6d48c6a15b4 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -5,6 +5,7 @@ module Gitlab module Importer class RepositoryImporter include Gitlab::ShellAdapter + include Gitlab::Utils::StrongMemoize attr_reader :project, :client, :wiki_formatter @@ -17,7 +18,7 @@ module Gitlab # Returns true if we should import the wiki for the project. # rubocop: disable CodeReuse/ActiveRecord def import_wiki? - client.repository(project.import_source)&.has_wiki && + client_repository&.has_wiki && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end @@ -52,6 +53,7 @@ module Gitlab refmap = Gitlab::GithubImport.refmap project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github') + project.change_head(default_branch) if default_branch true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e fail_import("Failed to import the repository: #{e.message}") @@ -82,6 +84,18 @@ module Gitlab project.import_state.mark_as_failed(message) false end + + private + + def default_branch + client_repository&.default_branch + end + + def client_repository + strong_memoize(:client_repository) do + client.repository(project.import_source) + end + end end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 435b74806e7..c2be7f3d63a 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -2,23 +2,38 @@ module Gitlab module GlRepository - def self.gl_repository(project, is_wiki) - "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" + PROJECT = RepoType.new( + name: :project, + access_checker_class: Gitlab::GitAccess, + repository_accessor: -> (project) { project.repository } + ).freeze + WIKI = RepoType.new( + name: :wiki, + access_checker_class: Gitlab::GitAccessWiki, + repository_accessor: -> (project) { project.wiki.repository } + ).freeze + + TYPES = { + PROJECT.name.to_s => PROJECT, + WIKI.name.to_s => WIKI + }.freeze + + def self.types + TYPES end - # rubocop: disable CodeReuse/ActiveRecord def self.parse(gl_repository) - match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) - unless match_data + type_name, _id = gl_repository.split('-').first + type = types[type_name] + subject_id = type&.fetch_id(gl_repository) + + unless subject_id raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" end - type, id = match_data.captures - project = Project.find_by(id: id) - wiki = type == 'wiki' + project = Project.find_by_id(subject_id) - [project, wiki] + [project, type] end - # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb new file mode 100644 index 00000000000..7abe6c29a25 --- /dev/null +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module GlRepository + class RepoType + attr_reader :name, + :access_checker_class, + :repository_accessor + + def initialize(name:, access_checker_class:, repository_accessor:) + @name = name + @access_checker_class = access_checker_class + @repository_accessor = repository_accessor + end + + def identifier_for_subject(subject) + "#{name}-#{subject.id}" + end + + def fetch_id(identifier) + match = /\A#{name}-(?<id>\d+)\z/.match(identifier) + match[:id] if match + end + + def wiki? + self == WIKI + end + + def project? + self == PROJECT + end + + def path_suffix + project? ? "" : ".#{name}" + end + + def repository_for(subject) + repository_accessor.call(subject) + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb index e3b9c59bd6e..18ea3a8d2f3 100644 --- a/lib/gitlab/grape_logging/loggers/perf_logger.rb +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -6,7 +6,10 @@ module Gitlab module Loggers class PerfLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) - { gitaly_calls: Gitlab::GitalyClient.get_request_count } + { + gitaly_calls: Gitlab::GitalyClient.get_request_count, + gitaly_duration: Gitlab::GitalyClient.query_time_ms + } end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb new file mode 100644 index 00000000000..7255293b194 --- /dev/null +++ b/lib/gitlab/group_search_results.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + class GroupSearchResults < SearchResults + def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) + super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) + + @group = group + end + + # rubocop:disable CodeReuse/ActiveRecord + def users + # 1: get all groups the current user has access to + groups = GroupsFinder.new(current_user).execute.joins(:users) + + # 2: Get the group's whole hierarchy + group_users = @group.direct_and_indirect_users + + # 3: get all users the current user has access to (-> + # `SearchResults#users`), which also applies the query. + users = super + + # 4: filter for users that belong to the previously selected groups + users + .where(id: group_users.select('id')) + .where(id: groups.select('members.user_id')) + end + # rubocop:enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index f5368d41629..1f0deebea39 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -97,7 +97,7 @@ module Gitlab def any_non_empty_queue?(*workers) workers.any? do |worker| - worker.jobs.any? + !Sidekiq::Queue.new(worker.queue).size.zero? end end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index c99353b9d49..d39ff8c21cc 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -48,7 +48,7 @@ module Gitlab } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) - .merge!(attrs) + .merge!(attrs) end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ad38e26e40a..d77b1d04644 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -55,7 +55,7 @@ module Gitlab } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) - .merge!(attrs) + .merge!(attrs) end end end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index fa3ff6c3f12..b3fe1fc0685 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -38,7 +38,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false) # These fields are set so we can create the correct merge request # diffs. @@ -47,24 +46,21 @@ module Gitlab merge_request.keep_around_commit + # We force to recreate all diffs to replace all existing data + # We use `.all` as otherwise `dependent: :nullify` (the default) + # takes an effect + merge_request.merge_request_diffs.all.delete_all if already_exists + # MR diffs normally use an "after_save" hook to pull data from Git. # All of this happens in the transaction started by calling # create/save/etc. This in turn can lead to these transactions being # held open for much longer than necessary. To work around this we # first save the diff, then populate it. - diff = - if already_exists - merge_request.merge_request_diffs.take || - merge_request.merge_request_diffs.build - else - merge_request.merge_request_diffs.build - end - + diff = merge_request.merge_request_diffs.build diff.importing = true diff.save diff.save_git_content end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index fa54fc17d95..89667976217 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -33,6 +33,7 @@ project_tree: - :user - merge_requests: - :metrics + - :suggestions - notes: - :author - events: @@ -117,6 +118,7 @@ excluded_attributes: - :description_html - :repository_languages - :bfg_object_map + - :tag_list namespaces: - :runners_token - :runners_token_encrypted diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 24daad638f4..e4bc437d787 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -80,8 +80,23 @@ module Gitlab # when the new_record? method incorrectly returns false. # # See https://gitlab.com/gitlab-org/gitlab-ee/issues/9903#note_145329964 - attributes = klass.attributes_builder.build_from_database(raw, {}) - klass.allocate.init_with("attributes" => attributes, "new_record" => new_record?(raw, klass)) + klass + .allocate + .init_with( + "attributes" => attributes_for(klass, raw), + "new_record" => new_record?(raw, klass) + ) + end + + def attributes_for(klass, raw) + # We have models that leave out some fields from the JSON export for + # security reasons, e.g. models that include the CacheMarkdownField. + # The ActiveRecord::AttributeSet we build from raw does know about + # these columns so we need manually set them. + missing_attributes = (klass.columns.map(&:name) - raw.keys) + missing_attributes.each { |column| raw[column] = nil } + + klass.attributes_builder.build_from_database(raw, {}) end def new_record?(raw, klass) diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index a9957a85d48..d46b5e3aee3 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -24,6 +24,30 @@ module Gitlab end end + # Filters an array of pods (as returned by the kubernetes API) by their annotations + def filter_by_annotation(items, annotations = {}) + items.select do |item| + metadata = item.fetch("metadata", {}) + item_annotations = metadata.fetch("annotations", nil) + next unless item_annotations + + annotations.all? { |k, v| item_annotations[k.to_s] == v } + end + end + + # Filters an array of pods (as returned by the kubernetes API) by their project and environment + def filter_by_project_environment(items, app, env) + pods = filter_by_annotation(items, { + 'app.gitlab.com/app' => app, + 'app.gitlab.com/env' => env + }) + return pods unless pods.empty? + + filter_by_label(items, { + 'app' => env, # deprecated: replaced by app.gitlab.com/env + }) + end + # Converts a pod (as returned by the kubernetes API) into a terminal def terminals_for_pod(api_url, namespace, pod) metadata = pod.fetch("metadata", {}) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index a68f8801c2a..58f06b6708c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -22,11 +22,17 @@ module Gitlab paginated_blobs(wiki_blobs, page) when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) + when 'users' + users.page(page).per(per_page) else super(scope, page, false) end end + def users + super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord + end + def blobs_count @blobs_count ||= blobs.count end diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index e7bfcb16582..93030fd454e 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -4,7 +4,7 @@ module Gitlab module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block, :warning + :condition_block, :parse_params_block, :action_block, :warning, :types def initialize(name, attributes = {}) @name = name @@ -17,6 +17,7 @@ module Gitlab @condition_block = attributes[:condition_block] @parse_params_block = attributes[:parse_params_block] @action_block = attributes[:action_block] + @types = attributes[:types] || [] end def all_names @@ -28,6 +29,7 @@ module Gitlab end def available?(context) + return false unless valid_type?(context) return true unless condition_block context.instance_exec(&condition_block) @@ -96,6 +98,10 @@ module Gitlab context.instance_exec(arg, &parse_params_block) end + + def valid_type?(context) + types.blank? || types.any? { |type| context.quick_action_target.is_a?(type) } + end end end end diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb new file mode 100644 index 00000000000..62c0fbb5afd --- /dev/null +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module CommitActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + # Commit only quick actions definitions + desc 'Tag this commit.' + explanation do |tag_name, message| + with_message = %{ with "#{message}"} if message.present? + "Tags this commit to #{tag_name}#{with_message}." + end + params 'v1.2.3 <message>' + parse_params do |tag_name_and_message| + tag_name_and_message.split(' ', 2) + end + types Commit + condition do + current_user.can?(:push_code, project) + end + command :tag do |tag_name, message| + @updates[:tag_name] = tag_name + @updates[:tag_message] = message + end + end + end + end +end diff --git a/lib/gitlab/quick_actions/common_actions.rb b/lib/gitlab/quick_actions/common_actions.rb new file mode 100644 index 00000000000..5d2732d4826 --- /dev/null +++ b/lib/gitlab/quick_actions/common_actions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module CommonActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + # This is a dummy command, so that it appears in the autocomplete commands + desc 'CC' + params '@user' + command :cc + end + end + end +end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index a3aab92061b..ecb2169151e 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -24,7 +24,7 @@ module Gitlab # Example: # # desc do - # "This is a dynamic description for #{noteable.to_ability_name}" + # "This is a dynamic description for #{quick_action_target.to_ability_name}" # end # command :command_key do |arguments| # # Awesome code block @@ -66,6 +66,23 @@ module Gitlab @explanation = block_given? ? block : text end + # Allows to define type(s) that must be met in order for the command + # to be returned by `.command_names` & `.command_definitions`. + # + # It is being evaluated before the conditions block is being evaluated + # + # If no types are passed then any type is allowed as the check is simply skipped. + # + # Example: + # + # types Commit, Issue, MergeRequest + # command :command_key do |arguments| + # # Awesome code block + # end + def types(*types_list) + @types = types_list + end + # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # It accepts a block that will be evaluated with the context @@ -144,7 +161,8 @@ module Gitlab params: @params, condition_block: @condition_block, parse_params_block: @parse_params_block, - action_block: block + action_block: block, + types: @types ) self.command_definitions << definition @@ -159,6 +177,7 @@ module Gitlab @condition_block = nil @warning = nil @parse_params_block = nil + @types = nil end end end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb new file mode 100644 index 00000000000..ad2e15d19fa --- /dev/null +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module IssuableActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + SHRUG = '¯\\＿(ツ)＿/¯'.freeze + TABLEFLIP = '(╯°□°)╯︵ ┻━┻'.freeze + + included do + # Issue, MergeRequest, Epic: quick actions definitions + desc do + "Close this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + end + explanation do + "Closes this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + end + types Issuable + condition do + quick_action_target.persisted? && + quick_action_target.open? && + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :close do + @updates[:state_event] = 'close' + end + + desc do + "Reopen this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + end + explanation do + "Reopens this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + end + types Issuable + condition do + quick_action_target.persisted? && + quick_action_target.closed? && + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :reopen do + @updates[:state_event] = 'reopen' + end + + desc 'Change title' + explanation do |title_param| + "Changes the title to \"#{title_param}\"." + end + params '<New title>' + types Issuable + condition do + quick_action_target.persisted? && + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :title do |title_param| + @updates[:title] = title_param + end + + desc 'Add label(s)' + explanation do |labels_param| + labels = find_label_references(labels_param) + + "Adds #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + end + params '~label1 ~"label 2"' + types Issuable + condition do + parent && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) && + find_labels.any? + end + command :label do |labels_param| + label_ids = find_label_ids(labels_param) + + if label_ids.any? + @updates[:add_label_ids] ||= [] + @updates[:add_label_ids] += label_ids + + @updates[:add_label_ids].uniq! + end + end + + desc 'Remove all or specific label(s)' + explanation do |labels_param = nil| + if labels_param.present? + labels = find_label_references(labels_param) + "Removes #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + else + 'Removes all labels.' + end + end + params '~label1 ~"label 2"' + types Issuable + condition do + quick_action_target.persisted? && + quick_action_target.labels.any? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) + end + command :unlabel do |labels_param = nil| + if labels_param.present? + label_ids = find_label_ids(labels_param) + + if label_ids.any? + @updates[:remove_label_ids] ||= [] + @updates[:remove_label_ids] += label_ids + + @updates[:remove_label_ids].uniq! + end + else + @updates[:label_ids] = [] + end + end + + desc 'Replace all label(s)' + explanation do |labels_param| + labels = find_label_references(labels_param) + "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + end + params '~label1 ~"label 2"' + types Issuable + condition do + quick_action_target.persisted? && + quick_action_target.labels.any? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) + end + command :relabel do |labels_param| + label_ids = find_label_ids(labels_param) + + if label_ids.any? + @updates[:label_ids] ||= [] + @updates[:label_ids] += label_ids + + @updates[:label_ids].uniq! + end + end + + desc 'Add a todo' + explanation 'Adds a todo.' + types Issuable + condition do + quick_action_target.persisted? && + !TodoService.new.todo_exist?(quick_action_target, current_user) + end + command :todo do + @updates[:todo_event] = 'add' + end + + desc 'Mark todo as done' + explanation 'Marks todo as done.' + types Issuable + condition do + quick_action_target.persisted? && + TodoService.new.todo_exist?(quick_action_target, current_user) + end + command :done do + @updates[:todo_event] = 'done' + end + + desc 'Subscribe' + explanation do + "Subscribes to this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + end + types Issuable + condition do + quick_action_target.persisted? && + !quick_action_target.subscribed?(current_user, project) + end + command :subscribe do + @updates[:subscription_event] = 'subscribe' + end + + desc 'Unsubscribe' + explanation do + "Unsubscribes from this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + end + types Issuable + condition do + quick_action_target.persisted? && + quick_action_target.subscribed?(current_user, project) + end + command :unsubscribe do + @updates[:subscription_event] = 'unsubscribe' + end + + desc 'Toggle emoji award' + explanation do |name| + "Toggles :#{name}: emoji award." if name + end + params ':emoji:' + types Issuable + condition do + quick_action_target.persisted? + end + parse_params do |emoji_param| + match = emoji_param.match(Banzai::Filter::EmojiFilter.emoji_pattern) + match[1] if match + end + command :award do |name| + if name && quick_action_target.user_can_award?(current_user) + @updates[:emoji_award] = name + end + end + + desc "Append the comment with #{SHRUG}" + params '<Comment>' + types Issuable + substitution :shrug do |comment| + "#{comment} #{SHRUG}" + end + + desc "Append the comment with #{TABLEFLIP}" + params '<Comment>' + types Issuable + substitution :tableflip do |comment| + "#{comment} #{TABLEFLIP}" + end + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb new file mode 100644 index 00000000000..1f08e8740a2 --- /dev/null +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module IssueActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + # Issue only quick actions definition + desc 'Set due date' + explanation do |due_date| + "Sets the due date to #{due_date.to_s(:medium)}." if due_date + end + params '<in 2 days | this Friday | December 31st>' + types Issue + condition do + quick_action_target.respond_to?(:due_date) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |due_date_param| + Chronic.parse(due_date_param).try(:to_date) + end + command :due do |due_date| + @updates[:due_date] = due_date if due_date + end + + desc 'Remove due date' + explanation 'Removes the due date.' + types Issue + condition do + quick_action_target.persisted? && + quick_action_target.respond_to?(:due_date) && + quick_action_target.due_date? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + command :remove_due_date do + @updates[:due_date] = nil + end + + desc 'Move issue from one column of the board to another' + explanation do |target_list_name| + label = find_label_references(target_list_name).first + "Moves issue to #{label} column in the board." if label + end + params '~"Target column"' + types Issue + condition do + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) && + quick_action_target.project.boards.count == 1 + end + # rubocop: disable CodeReuse/ActiveRecord + command :board_move do |target_list_name| + label_ids = find_label_ids(target_list_name) + + if label_ids.size == 1 + label_id = label_ids.first + + # Ensure this label corresponds to a list on the board + next unless Label.on_project_boards(quick_action_target.project_id).where(id: label_id).exists? + + @updates[:remove_label_ids] = + quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) + @updates[:add_label_ids] = [label_id] + end + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Mark this issue as a duplicate of another issue' + explanation do |duplicate_reference| + "Marks this issue as a duplicate of #{duplicate_reference}." + end + params '#issue' + types Issue + condition do + quick_action_target.persisted? && + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :duplicate do |duplicate_param| + canonical_issue = extract_references(duplicate_param, :issue).first + + if canonical_issue.present? + @updates[:canonical_issue_id] = canonical_issue.id + end + end + + desc 'Move this issue to another project.' + explanation do |path_to_project| + "Moves this issue to #{path_to_project}." + end + params 'path/to/project' + types Issue + condition do + quick_action_target.persisted? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + command :move do |target_project_path| + target_project = Project.find_by_full_path(target_project_path) + + if target_project.present? + @updates[:target_project] = target_project + end + end + + desc 'Make issue confidential.' + explanation do + 'Makes this issue confidential' + end + types Issue + condition do + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :confidential do + @updates[:confidential] = true + end + + desc 'Create a merge request.' + explanation do |branch_name = nil| + branch_text = branch_name ? "branch '#{branch_name}'" : 'a branch' + "Creates #{branch_text} and a merge request to resolve this issue" + end + params "<branch name>" + types Issue + condition do + current_user.can?(:create_merge_request_in, project) && current_user.can?(:push_code, project) + end + command :create_merge_request do |branch_name = nil| + @updates[:create_merge_request] = { + branch_name: branch_name, + issue_iid: quick_action_target.iid + } + end + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb new file mode 100644 index 00000000000..08872eda410 --- /dev/null +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module IssueAndMergeRequestActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + # Issue, MergeRequest: quick actions definitions + desc 'Assign' + # rubocop: disable CodeReuse/ActiveRecord + explanation do |users| + users = quick_action_target.allows_multiple_assignees? ? users : users.take(1) + "Assigns #{users.map(&:to_reference).to_sentence}." + end + # rubocop: enable CodeReuse/ActiveRecord + params do + quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '@user' + end + types Issue, MergeRequest + condition do + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |assignee_param| + extract_users(assignee_param) + end + command :assign do |users| + next if users.empty? + + if quick_action_target.allows_multiple_assignees? + @updates[:assignee_ids] ||= quick_action_target.assignees.map(&:id) + @updates[:assignee_ids] += users.map(&:id) + else + @updates[:assignee_ids] = [users.first.id] + end + end + + desc do + if quick_action_target.allows_multiple_assignees? + 'Remove all or specific assignee(s)' + else + 'Remove assignee' + end + end + explanation do |users = nil| + assignees = quick_action_target.assignees + assignees &= users if users.present? && quick_action_target.allows_multiple_assignees? + "Removes #{'assignee'.pluralize(assignees.size)} #{assignees.map(&:to_reference).to_sentence}." + end + params do + quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '' + end + types Issue, MergeRequest + condition do + quick_action_target.persisted? && + quick_action_target.assignees.any? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |unassign_param| + # When multiple users are assigned, all will be unassigned if multiple assignees are no longer allowed + extract_users(unassign_param) if quick_action_target.allows_multiple_assignees? + end + command :unassign do |users = nil| + if quick_action_target.allows_multiple_assignees? && users&.any? + @updates[:assignee_ids] ||= quick_action_target.assignees.map(&:id) + @updates[:assignee_ids] -= users.map(&:id) + else + @updates[:assignee_ids] = [] + end + end + + desc 'Set milestone' + explanation do |milestone| + "Sets the milestone to #{milestone.to_reference}." if milestone + end + params '%"milestone"' + types Issue, MergeRequest + condition do + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) && + find_milestones(project, state: 'active').any? + end + parse_params do |milestone_param| + extract_references(milestone_param, :milestone).first || + find_milestones(project, title: milestone_param.strip).first + end + command :milestone do |milestone| + @updates[:milestone_id] = milestone.id if milestone + end + + desc 'Remove milestone' + explanation do + "Removes #{quick_action_target.milestone.to_reference(format: :name)} milestone." + end + types Issue, MergeRequest + condition do + quick_action_target.persisted? && + quick_action_target.milestone_id? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + command :remove_milestone do + @updates[:milestone_id] = nil + end + + desc 'Copy labels and milestone from other issue or merge request' + explanation do |source_issuable| + "Copy labels and milestone from #{source_issuable.to_reference}." + end + params '#issue | !merge_request' + types Issue, MergeRequest + condition do + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + parse_params do |issuable_param| + extract_references(issuable_param, :issue).first || + extract_references(issuable_param, :merge_request).first + end + command :copy_metadata do |source_issuable| + if source_issuable.present? && source_issuable.project.id == quick_action_target.project.id + @updates[:add_label_ids] = source_issuable.labels.map(&:id) + @updates[:milestone_id] = source_issuable.milestone.id if source_issuable.milestone + end + end + + desc 'Set time estimate' + explanation do |time_estimate| + time_estimate = Gitlab::TimeTrackingFormatter.output(time_estimate) + + "Sets time estimate to #{time_estimate}." if time_estimate + end + params '<1w 3d 2h 14m>' + types Issue, MergeRequest + condition do + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |raw_duration| + Gitlab::TimeTrackingFormatter.parse(raw_duration) + end + command :estimate do |time_estimate| + if time_estimate + @updates[:time_estimate] = time_estimate + end + end + + desc 'Add or subtract spent time' + explanation do |time_spent, time_spent_date| + if time_spent + if time_spent > 0 + verb = 'Adds' + value = time_spent + else + verb = 'Subtracts' + value = -time_spent + end + + "#{verb} #{Gitlab::TimeTrackingFormatter.output(value)} spent time." + end + end + params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' + types Issue, MergeRequest + condition do + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + parse_params do |raw_time_date| + Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute + end + command :spend do |time_spent, time_spent_date| + if time_spent + @updates[:spend_time] = { + duration: time_spent, + user_id: current_user.id, + spent_at: time_spent_date + } + end + end + + desc 'Remove time estimate' + explanation 'Removes time estimate.' + types Issue, MergeRequest + condition do + quick_action_target.persisted? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + command :remove_estimate do + @updates[:time_estimate] = 0 + end + + desc 'Remove spent time' + explanation 'Removes spent time.' + condition do + quick_action_target.persisted? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + types Issue, MergeRequest + command :remove_time_spent do + @updates[:spend_time] = { duration: :reset, user_id: current_user.id } + end + + desc "Lock the discussion" + explanation "Locks the discussion" + types Issue, MergeRequest + condition do + quick_action_target.persisted? && + !quick_action_target.discussion_locked? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :lock do + @updates[:discussion_locked] = true + end + + desc "Unlock the discussion" + explanation "Unlocks the discussion" + types Issue, MergeRequest + condition do + quick_action_target.persisted? && + quick_action_target.discussion_locked? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :unlock do + @updates[:discussion_locked] = false + end + end + end + end +end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb new file mode 100644 index 00000000000..bade59182a1 --- /dev/null +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module MergeRequestActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + # MergeRequest only quick actions definitions + desc 'Merge (when the pipeline succeeds)' + explanation 'Merges this merge request when the pipeline succeeds.' + types MergeRequest + condition do + last_diff_sha = params && params[:merge_request_diff_head_sha] + quick_action_target.persisted? && + quick_action_target.mergeable_with_quick_action?(current_user, autocomplete_precheck: !last_diff_sha, last_diff_sha: last_diff_sha) + end + command :merge do + @updates[:merge] = params[:merge_request_diff_head_sha] + end + + desc 'Toggle the Work In Progress status' + explanation do + verb = quick_action_target.work_in_progress? ? 'Unmarks' : 'Marks' + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + "#{verb} this #{noun} as Work In Progress." + end + types MergeRequest + condition do + quick_action_target.respond_to?(:work_in_progress?) && + # Allow it to mark as WIP on MR creation page _or_ through MR notes. + (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) + end + command :wip do + @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' + end + + desc 'Set target branch' + explanation do |branch_name| + "Sets target branch to #{branch_name}." + end + params '<Local branch name>' + types MergeRequest + condition do + quick_action_target.respond_to?(:target_branch) && + (current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) || + quick_action_target.new_record?) + end + parse_params do |target_branch_param| + target_branch_param.strip + end + command :target_branch do |branch_name| + @updates[:target_branch] = branch_name if project.repository.branch_exists?(branch_name) + end + end + end + end +end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 202d310e237..207a80b7db2 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -5,19 +5,26 @@ module Gitlab NotFoundError = Class.new(StandardError) def self.parse(repo_path) - wiki = false project_path = repo_path.sub(/\.git\z/, '').sub(%r{\A/}, '') - project, was_redirected = find_project(project_path) - - if project_path.end_with?('.wiki') && project.nil? - project, was_redirected = find_project(project_path.chomp('.wiki')) - wiki = true + # Detect the repo type based on the path, the first one tried is the project + # type, which does not have a suffix. + Gitlab::GlRepository.types.each do |_name, type| + # If the project path does not end with the defined suffix, try the next + # type. + # We'll always try to find a project with an empty suffix (for the + # `Gitlab::GlRepository::PROJECT` type. + next unless project_path.end_with?(type.path_suffix) + + project, was_redirected = find_project(project_path.chomp(type.path_suffix)) + redirected_path = project_path if was_redirected + + # If we found a matching project, then the type was matched, no need to + # continue looking. + return [project, type, redirected_path] if project end - redirected_path = project_path if was_redirected - - [project, wiki, redirected_path] + nil end def self.find_project(project_path) diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 491148ec1a6..8988b9ad7be 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -32,6 +32,8 @@ module Gitlab merge_requests.page(page).per(per_page) when 'milestones' milestones.page(page).per(per_page) + when 'users' + users.page(page).per(per_page) else Kaminari.paginate_array([]).page(page).per(per_page) end @@ -71,6 +73,12 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord + def limited_users_count + @limited_users_count ||= users.limit(count_limit).count + end + # rubocop:enable CodeReuse/ActiveRecord + def single_commit_result? false end @@ -79,6 +87,12 @@ module Gitlab 1001 end + def users + return User.none unless Ability.allowed?(current_user, :read_users_list) + + UsersFinder.new(current_user, search: query).execute + end + private def projects diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 40b641b8317..93182607616 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -10,18 +10,6 @@ module Gitlab Error = Class.new(StandardError) - KeyAdder = Struct.new(:io) do - def add_key(id, key) - key = Gitlab::Shell.strip_key(key) - # Newline and tab are part of the 'protocol' used to transmit id+key to the other end - if key.include?("\t") || key.include?("\n") - raise Error.new("Invalid key: #{key.inspect}") - end - - io.puts("#{id}\t#{key}") - end - end - class << self def secret_token @secret_token ||= begin @@ -40,10 +28,6 @@ module Gitlab .join('GITLAB_SHELL_VERSION')).strip end - def strip_key(key) - key.split(/[ ]+/)[0, 2].join(' ') - end - private # Create (if necessary) and link the secret token file @@ -173,7 +157,7 @@ module Gitlab false end - # Add new key to gitlab-shell + # Add new key to authorized_keys # # Ex. # add_key("key-42", "sha-rsa ...") @@ -181,33 +165,53 @@ module Gitlab def add_key(key_id, key_content) return unless self.authorized_keys_enabled? - gitlab_shell_fast_execute([gitlab_shell_keys_path, - 'add-key', key_id, self.class.strip_key(key_content)]) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([ + gitlab_shell_keys_path, + 'add-key', + key_id, + strip_key(key_content) + ]) + else + gitlab_authorized_keys.add_key(key_id, key_content) + end end # Batch-add keys to authorized_keys # # Ex. - # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } - def batch_add_keys(&block) + # batch_add_keys(Key.all) + def batch_add_keys(keys) return unless self.authorized_keys_enabled? - IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| - yield(KeyAdder.new(io)) + if shell_out_for_gitlab_keys? + begin + IO.popen("#{gitlab_shell_keys_path} batch-add-keys", 'w') do |io| + add_keys_to_io(keys, io) + end + + $?.success? + rescue Error + false + end + else + gitlab_authorized_keys.batch_add_keys(keys) end end - # Remove ssh key from gitlab shell + # Remove ssh key from authorized_keys # # Ex. - # remove_key("key-342", "sha-rsa ...") + # remove_key("key-342") # - def remove_key(key_id, key_content = nil) + def remove_key(id, _ = nil) return unless self.authorized_keys_enabled? - args = [gitlab_shell_keys_path, 'rm-key', key_id] - args << key_content if key_content - gitlab_shell_fast_execute(args) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'rm-key', id]) + else + gitlab_authorized_keys.rm_key(id) + end end # Remove all ssh keys from gitlab shell @@ -218,7 +222,11 @@ module Gitlab def remove_all_keys return unless self.authorized_keys_enabled? - gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) + else + gitlab_authorized_keys.clear + end end # Remove ssh keys from gitlab shell that are not in the DB @@ -247,33 +255,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Iterate over all ssh key IDs from gitlab shell, in batches - # - # Ex. - # batch_read_key_ids { |batch| keys = Key.where(id: batch) } - # - def batch_read_key_ids(batch_size: 100, &block) - return unless self.authorized_keys_enabled? - - list_key_ids do |key_id_stream| - key_id_stream.lazy.each_slice(batch_size) do |lines| - key_ids = lines.map { |l| l.chomp.to_i } - yield(key_ids) - end - end - end - - # Stream all ssh key IDs from gitlab shell, separated by newlines - # - # Ex. - # list_key_ids - # - def list_key_ids(&block) - return unless self.authorized_keys_enabled? - - IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) - end - # Add empty directory for storing repositories # # Ex. @@ -378,6 +359,10 @@ module Gitlab private + def shell_out_for_gitlab_keys? + Gitlab.config.gitlab_shell.authorized_keys_file.blank? + end + def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -415,6 +400,40 @@ module Gitlab raise Error, e end + def gitlab_authorized_keys + @gitlab_authorized_keys ||= Gitlab::AuthorizedKeys.new + end + + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + if shell_out_for_gitlab_keys? + IO.popen("#{gitlab_shell_keys_path} list-key-ids") do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + yield(lines.map { |l| l.chomp.to_i }) + end + end + else + gitlab_authorized_keys.list_key_ids.lazy.each_slice(batch_size) do |key_ids| + yield(key_ids) + end + end + end + + def strip_key(key) + key.split(/[ ]+/)[0, 2].join(' ') + end + + def add_keys_to_io(keys, io) + keys.each do |k| + key = strip_key(k.key) + + raise Error.new("Invalid key: #{key.inspect}") if key.include?("\t") || key.include?("\n") + + io.puts("#{k.shell_id}\t#{key}") + end + end + class GitalyGitlabProjects attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 3b8de64913b..fb303e3fb0c 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -48,7 +48,9 @@ module Gitlab end def self.workers - @workers ||= find_workers(Rails.root.join('app', 'workers')) + @workers ||= + find_workers(Rails.root.join('app', 'workers')) + + find_workers(Rails.root.join('ee', 'app', 'workers')) end def self.find_workers(root) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f9c9ea4f936..75477c7cf18 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -84,6 +84,7 @@ module Gitlab projects: count(Project), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), + projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), protected_branches: count(ProtectedBranch), releases: count(Release), remote_mirrors: count(RemoteMirror), diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index b41d085ee77..f0557f6ad68 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -11,7 +11,9 @@ module Gitlab USERNAME_REGEXP = User.reference_pattern def initialize(text) - @text = text + # EE passes an Array to `text` in a few places, so we want to support both + # here. + @text = Array(text).join(' ') end def users diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 99fa65e0e90..16ec8a8bb28 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -104,6 +104,12 @@ module Gitlab nil end + def try_megabytes_to_bytes(size) + Integer(size).megabytes + rescue ArgumentError + size + end + def bytes_to_megabytes(bytes) bytes.to_f / Numeric::MEGABYTE end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 265f6213a99..5d5a867c9ab 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -20,14 +20,14 @@ module Gitlab SECRET_LENGTH = 32 class << self - def git_http_ok(repository, is_wiki, user, action, show_all_refs: false) + def git_http_ok(repository, repo_type, user, action, show_all_refs: false) raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) project = repository.project attrs = { GL_ID: Gitlab::GlId.gl_id(user), - GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + GL_REPOSITORY: repo_type.identifier_for_subject(project), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 49ec196b103..bb1aa2a7a10 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -3,7 +3,7 @@ module Sentry class Client Error = Class.new(StandardError) - SentryError = Class.new(StandardError) + MissingKeysError = Class.new(StandardError) attr_accessor :url, :token @@ -14,18 +14,29 @@ module Sentry def list_issues(issue_status:, limit:) issues = get_issues(issue_status: issue_status, limit: limit) - map_to_errors(issues) + + handle_mapping_exceptions do + map_to_errors(issues) + end end def list_projects projects = get_projects - map_to_projects(projects) - rescue KeyError => e - raise Client::SentryError, "Sentry API response is missing keys. #{e.message}" + + handle_mapping_exceptions do + map_to_projects(projects) + end end private + def handle_mapping_exceptions(&block) + yield + rescue KeyError => e + Gitlab::Sentry.track_acceptable_exception(e) + raise Client::MissingKeysError, "Sentry API response is missing keys. #{e.message}" + end + def request_params { headers: { @@ -57,7 +68,7 @@ module Sentry raise Client::Error, "Sentry response status code: #{response.code}" end - response.as_json + response end def projects_api_url @@ -94,7 +105,6 @@ module Sentry def map_to_error(issue) id = issue.fetch('id') - project = issue.fetch('project') count = issue.fetch('count', nil) @@ -117,9 +127,9 @@ module Sentry short_id: issue.fetch('shortId', nil), status: issue.fetch('status', nil), frequency: frequency, - project_id: project.fetch('id'), - project_name: project.fetch('name', nil), - project_slug: project.fetch('slug', nil) + project_id: issue.dig('project', 'id'), + project_name: issue.dig('project', 'name'), + project_slug: issue.dig('project', 'slug') ) end @@ -127,12 +137,12 @@ module Sentry organization = project.fetch('organization') Gitlab::ErrorTracking::Project.new( - id: project.fetch('id'), + id: project.fetch('id', nil), name: project.fetch('name'), slug: project.fetch('slug'), status: project.dig('status'), organization_name: organization.fetch('name'), - organization_id: organization.fetch('id'), + organization_id: organization.fetch('id', nil), organization_slug: organization.fetch('slug') ) end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index b8798fb3cfd..7872e5b08c0 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -34,9 +34,6 @@ namespace :gitlab do puts "Sidekiq Version:#{Sidekiq::VERSION}" puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" - # check database adapter - database_adapter = ActiveRecord::Base.connection.adapter_name.downcase - project = Group.new(path: "some-group").projects.build(path: "some-project") # construct clone URLs http_clone_url = project.http_url_to_repo @@ -49,7 +46,8 @@ namespace :gitlab do puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab.revision}" puts "Directory:\t#{Rails.root}" - puts "DB Adapter:\t#{database_adapter}" + puts "DB Adapter:\t#{Gitlab::Database.human_adapter_name}" + puts "DB Version:\t#{Gitlab::Database.version}" puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 0ebc6f00793..ee3ef9dad6e 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -103,19 +103,12 @@ namespace :gitlab do Gitlab::Shell.new.remove_all_keys - Gitlab::Shell.new.batch_add_keys do |adder| - Key.find_each(batch_size: 1000) do |key| - adder.add_key(key.shell_id, key.key) - print '.' + Key.find_in_batches(batch_size: 1000) do |keys| + unless Gitlab::Shell.new.batch_add_keys(keys) + puts "Failed to add keys...".color(:red) + exit 1 end end - puts "" - - unless $?.success? - puts "Failed to add keys...".color(:red) - exit 1 - end - rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) exit 1 diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 53325d492d1..02987f2beef 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,13 +1,24 @@ unless Rails.env.production? namespace :karma do desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + task fixtures: ['karma:copy_emojis_from_public_folder', 'karma:rspec_fixtures'] + + desc 'GitLab | Karma | Generate fixtures using RSpec' + RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' end + desc 'GitLab | Karma | Copy emojis file' + task :copy_emojis_from_public_folder do + # Copying the emojis.json from the public folder + fixture_file_name = Rails.root.join('spec/javascripts/fixtures/emojis/emojis.json') + FileUtils.mkdir_p(File.dirname(fixture_file_name)) + FileUtils.cp(Rails.root.join('public/-/emojis/1/emojis.json'), fixture_file_name) + end + desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do sh "yarn run karma" do |ok, res| |