diff options
Diffstat (limited to 'lib')
123 files changed, 2529 insertions, 322 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index e500a93b31e..219ed45eff6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -18,7 +18,7 @@ module API formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), - GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index a1851ba3627..89b7e5c5e4b 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -69,12 +69,12 @@ module API post endpoint do not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? - award = awardable.create_award_emoji(params[:name], current_user) + service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute - if award.persisted? - present award, with: Entities::AwardEmoji + if service[:status] == :success + present service[:award], with: Entities::AwardEmoji else - not_found!("Award Emoji #{award.errors.messages}") + not_found!("Award Emoji #{service[:message]}") end end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 6c1acc3963f..9125207167c 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -239,7 +239,7 @@ module API # because notes are redacted if they point to projects that # cannot be accessed by the user. notes = prepare_notes_for_rendering(notes) - notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + notes.select { |n| n.visible_for?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 54f0c22d484..44de795cf52 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -647,7 +647,10 @@ module API end end - expose :subscribed do |issue, options| + # Calculating the value of subscribed field triggers Markdown + # processing. We can't do that for multiple issues / merge + # requests in a single API request. + expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end end @@ -1173,6 +1176,9 @@ module API attributes.delete(:performance_bar_enabled) attributes.delete(:allow_local_requests_from_hooks_and_services) + # let's not expose the secret key in a response + attributes.delete(:asset_proxy_secret_key) + attributes end @@ -1704,5 +1710,35 @@ module API class ClusterGroup < Cluster expose :group, using: Entities::BasicGroupDetails end + + module InternalPostReceive + class Message < Grape::Entity + expose :message + expose :type + end + + class Response < Grape::Entity + expose :messages, using: Message + expose :reference_counter_decreased + end + end end end + +# rubocop: disable Cop/InjectEnterpriseEditionModule +API::Entities.prepend_if_ee('EE::API::Entities::Entities') +::API::Entities::ApplicationSetting.prepend_if_ee('EE::API::Entities::ApplicationSetting') +::API::Entities::Board.prepend_if_ee('EE::API::Entities::Board') +::API::Entities::Group.prepend_if_ee('EE::API::Entities::Group', with_descendants: true) +::API::Entities::GroupDetail.prepend_if_ee('EE::API::Entities::GroupDetail') +::API::Entities::IssueBasic.prepend_if_ee('EE::API::Entities::IssueBasic', with_descendants: true) +::API::Entities::List.prepend_if_ee('EE::API::Entities::List') +::API::Entities::MergeRequestBasic.prepend_if_ee('EE::API::Entities::MergeRequestBasic', with_descendants: true) +::API::Entities::Namespace.prepend_if_ee('EE::API::Entities::Namespace') +::API::Entities::Project.prepend_if_ee('EE::API::Entities::Project', with_descendants: true) +::API::Entities::ProtectedRefAccess.prepend_if_ee('EE::API::Entities::ProtectedRefAccess') +::API::Entities::UserPublic.prepend_if_ee('EE::API::Entities::UserPublic', with_descendants: true) +::API::Entities::Todo.prepend_if_ee('EE::API::Entities::Todo') +::API::Entities::ProtectedBranch.prepend_if_ee('EE::API::Entities::ProtectedBranch') +::API::Entities::Identity.prepend_if_ee('EE::API::Entities::Identity') +::API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin', with_descendants: true) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index f545f33c06b..0bcd09d3977 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -216,6 +216,7 @@ module API use :pagination use :with_custom_attributes + use :optional_projects_params end get ":id/projects" do projects = find_group_projects(params) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1aa6dc44bf7..5755f4b8d74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -416,17 +416,7 @@ module API # rubocop: enable CodeReuse/ActiveRecord def project_finder_params - finder_params = { without_deleted: true } - finder_params[:owned] = true if params[:owned].present? - finder_params[:non_public] = true if params[:membership].present? - finder_params[:starred] = true if params[:starred].present? - finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] - finder_params[:archived] = archived_param unless params[:archived].nil? - finder_params[:search] = params[:search] if params[:search] - finder_params[:user] = params.delete(:user) if params[:user] - finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] - finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] - finder_params + project_finder_params_ce.merge(project_finder_params_ee) end # file helpers @@ -461,6 +451,27 @@ module API end end + protected + + def project_finder_params_ce + finder_params = { without_deleted: true } + finder_params[:owned] = true if params[:owned].present? + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = archived_param unless params[:archived].nil? + finder_params[:search] = params[:search] if params[:search] + finder_params[:user] = params.delete(:user) if params[:user] + finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] + finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] + finder_params + end + + # Overridden in EE + def project_finder_params_ee + {} + end + private # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index 2c33d79f6c8..6af12828ca5 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -28,6 +28,13 @@ module API use :optional_params_ce use :optional_params_ee end + + params :optional_projects_params_ee do + end + + params :optional_projects_params do + use :optional_projects_params_ee + end end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 9afe6c5b027..6b438235258 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -44,8 +44,6 @@ module API end def process_mr_push_options(push_options, project, user, changes) - output = {} - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/61359') service = ::MergeRequests::PushOptionsHandlerService.new( @@ -56,15 +54,13 @@ module API ).execute if service.errors.present? - output[:warnings] = push_options_warning(service.errors.join("\n\n")) + push_options_warning(service.errors.join("\n\n")) end - - output end def push_options_warning(warning) options = Array.wrap(params[:push_options]).map { |p| "'#{p}'" }.join(' ') - "Error encountered with push options #{options}: #{warning}" + "WARNINGS:\nError encountered with push options #{options}: #{warning}" end def redis_ping diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index 5b7199fddb0..a8480bb9339 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -27,6 +27,10 @@ module API ] end + def self.sort_options + %w[created_at updated_at priority due_date relative_position label_priority milestone_due popularity] + end + def issue_finder(args = {}) args = declared_params.merge(args) @@ -34,15 +38,14 @@ module API args[:milestone_title] ||= args.delete(:milestone) args[:label_name] ||= args.delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] + args[:sort] = "#{args[:order_by]}_#{args[:sort]}" IssuesFinder.new(current_user, args) end def find_issues(args = {}) finder = issue_finder(args) - issues = finder.execute.with_api_entity_associations - - issues.reorder(order_options_with_tie_breaker) # rubocop: disable CodeReuse/ActiveRecord + finder.execute.with_api_entity_associations end def issues_statistics(args = {}) diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index 896b0aba52b..ec5b688dd1c 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -11,9 +11,9 @@ module API optional :description, type: String, desc: 'The description of label to be created' end - def find_label(parent, id, include_ancestor_groups: true) + def find_label(parent, id_or_title, include_ancestor_groups: true) labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups) - label = labels.find_by_id(id) || labels.find_by_title(id) + label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title) label || not_found!('Label') end @@ -35,12 +35,7 @@ module API priority = params.delete(:priority) label_params = declared_params(include_missing: false) - label = - if parent.is_a?(Project) - ::Labels::CreateService.new(label_params).execute(project: parent) - else - ::Labels::CreateService.new(label_params).execute(group: parent) - end + label = ::Labels::CreateService.new(label_params).execute(create_service_params(parent)) if label.persisted? if parent.is_a?(Project) @@ -56,10 +51,13 @@ module API def update_label(parent, entity) authorize! :admin_label, parent - label = find_label(parent, params[:name], include_ancestor_groups: false) + label = find_label(parent, params_id_or_title, include_ancestor_groups: false) update_priority = params.key?(:priority) priority = params.delete(:priority) + # params is used to update the label so we need to remove this field here + params.delete(:label_id) + label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label) render_validation_error!(label) unless label.valid? @@ -77,10 +75,24 @@ module API def delete_label(parent) authorize! :admin_label, parent - label = find_label(parent, params[:name], include_ancestor_groups: false) + label = find_label(parent, params_id_or_title, include_ancestor_groups: false) destroy_conditionally!(label) end + + def params_id_or_title + @params_id_or_title ||= params[:label_id] || params[:name] + end + + def create_service_params(parent) + if parent.is_a?(Project) + { project: parent } + elsif parent.is_a?(Group) + { group: parent } + else + raise TypeError, 'Parent type is not supported' + end + end end end end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 6bf9057fad7..f445834323d 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -3,6 +3,8 @@ module API module Helpers module NotesHelpers + include ::RendersNotes + def self.noteable_types # This is a method instead of a constant, allowing EE to more easily # extend it. @@ -10,7 +12,7 @@ module API end def update_note(noteable, note_id) - note = noteable.notes.find(params[:note_id]) + note = noteable.notes.find(note_id) authorize! :admin_note, note @@ -59,8 +61,8 @@ module API end def get_note(noteable, note_id) - note = noteable.notes.with_metadata.find(params[:note_id]) - can_read_note = !note.cross_reference_not_visible_for?(current_user) + note = noteable.notes.with_metadata.find(note_id) + can_read_note = note.visible_for?(current_user) if can_read_note present note, with: Entities::Note diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 224aaaaf006..088ea5bd79a 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -256,25 +256,26 @@ module API post '/post_receive' do status 200 - output = {} # Messages to gitlab-shell + response = Gitlab::InternalPostReceive::Response.new user = identify(params[:identifier]) project = Gitlab::GlRepository.parse(params[:gl_repository]).first push_options = Gitlab::PushOptions.new(params[:push_options]) + response.reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease + PostReceive.perform_async(params[:gl_repository], params[:identifier], params[:changes], push_options.as_json) mr_options = push_options.get(:merge_request) - output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? + if mr_options.present? + message = process_mr_push_options(mr_options, project, user, params[:changes]) + response.add_alert_message(message) + end broadcast_message = BroadcastMessage.current&.last&.message - reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease + response.add_alert_message(broadcast_message) - output.merge!( - broadcast_message: broadcast_message, - reference_counter_decreased: reference_counter_decreased, - merge_request_urls: merge_request_urls - ) + response.add_merge_request_urls(merge_request_urls) # A user is not guaranteed to be returned; an orphaned write deploy # key could be used @@ -282,11 +283,11 @@ module API redirect_message = Gitlab::Checks::ProjectMoved.fetch_message(user.id, project.id) project_created_message = Gitlab::Checks::ProjectCreated.fetch_message(user.id, project.id) - output[:redirected_message] = redirect_message if redirect_message - output[:project_created_message] = project_created_message if project_created_message + response.add_basic_message(redirect_message) + response.add_basic_message(project_created_message) end - output + present response, with: Entities::InternalPostReceive::Response end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d687acf3423..215178478d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -44,7 +44,7 @@ module API optional :with_labels_details, type: Boolean, desc: 'Return more label data than just lable title', default: false optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + optional :order_by, type: String, values: Helpers::IssuesHelpers.sort_options, default: 'created_at', desc: 'Return issues ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return issues sorted in `asc` or `desc` order.' @@ -96,7 +96,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -122,7 +123,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -161,7 +163,8 @@ module API with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c183198d3c6..de89e94b0c0 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -38,11 +38,13 @@ module API success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be updated' + optional :label_id, type: Integer, desc: 'The id of the label to be updated' + optional :name, type: String, desc: 'The name of the label to be updated' optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The new description of label' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + exactly_one_of :label_id, :name at_least_one_of :new_name, :color, :description, :priority end put ':id/labels' do @@ -53,11 +55,38 @@ module API success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be deleted' + optional :label_id, type: Integer, desc: 'The id of the label to be deleted' + optional :name, type: String, desc: 'The name of the label to be deleted' + exactly_one_of :label_id, :name end delete ':id/labels' do delete_label(user_project) end + + desc 'Promote a label to a group label' do + detail 'This feature was added in GitLab 12.3' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be promoted' + end + put ':id/labels/promote' do + authorize! :admin_label, user_project + + label = find_label(user_project, params[:name], include_ancestor_groups: false) + + begin + group_label = ::Labels::PromoteService.new(user_project, current_user).execute(label) + + if group_label + present group_label, with: Entities::GroupLabel, current_user: current_user, parent: user_project.group + else + render_api_error!('Failed to promote project label to group label', 400) + end + rescue => error + render_api_error!(error.to_s, 400) + end + end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 9381f045144..16fca9acccb 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -36,12 +36,13 @@ module API # page can have less elements than :per_page even if # there's more than one page. raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker) - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(raw_notes) - .reject { |n| n.cross_reference_not_visible_for?(current_user) } + + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + notes = paginate(raw_notes) + notes = prepare_notes_for_rendering(notes) + notes = notes.select { |note| note.visible_for?(current_user) } present notes, with: Entities::Note end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 667bf1ec801..9e888368e7b 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -4,7 +4,7 @@ module API class Pipelines < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } params do requires :id, type: String, desc: 'The project ID' @@ -32,6 +32,7 @@ module API end get ':id/pipelines' do authorize! :read_pipeline, user_project + authorize! :read_build, user_project pipelines = PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index a607df411a6..b4545295d54 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -51,16 +51,18 @@ module API params do requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' - requires :code, type: String, allow_blank: false, desc: 'The content of the snippet' + optional :code, type: String, allow_blank: false, desc: 'The content of the snippet (deprecated in favor of "content")' + optional :content, type: String, allow_blank: false, desc: 'The content of the snippet' optional :description, type: String, desc: 'The description of a snippet' requires :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' + mutually_exclusive :code, :content end post ":id/snippets" do authorize! :create_project_snippet, user_project - snippet_params = declared_params.merge(request: request, api: true) - snippet_params[:content] = snippet_params.delete(:code) + snippet_params = declared_params(include_missing: false).merge(request: request, api: true) + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute @@ -80,12 +82,14 @@ module API requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' - optional :code, type: String, allow_blank: false, desc: 'The content of the snippet' + optional :code, type: String, allow_blank: false, desc: 'The content of the snippet (deprecated in favor of "content")' + optional :content, type: String, allow_blank: false, desc: 'The content of the snippet' optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' - at_least_one_of :title, :file_name, :code, :visibility_level + at_least_one_of :title, :file_name, :code, :content, :visibility_level + mutually_exclusive :code, :content end # rubocop: disable CodeReuse/ActiveRecord put ":id/snippets/:snippet_id" do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3c0f49eb84a..55141fd22fd 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -496,11 +496,13 @@ module API end params do optional :search, type: String, desc: 'Return list of users matching the search criteria' + optional :skip_users, type: Array[Integer], desc: 'Filter out users with the specified IDs' use :pagination end get ':id/users' do users = DeclarativePolicy.subject_scope { user_project.team.users } users = users.search(params[:search]) if params[:search].present? + users = users.where_not_in(params[:skip_users]) if params[:skip_users].present? present paginate(users), with: Entities::UserBasic end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c36ee5af63f..dd27ebab83d 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -36,6 +36,10 @@ module API given akismet_enabled: ->(val) { val } do requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' end + optional :asset_proxy_enabled, type: Boolean, desc: 'Enable proxying of assets' + optional :asset_proxy_url, type: String, desc: 'URL of the asset proxy server' + optional :asset_proxy_secret_key, type: String, desc: 'Shared secret with the asset proxy server' + optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' @@ -104,6 +108,11 @@ module API requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' end + optional :login_recaptcha_protection_enabled, type: Boolean, desc: 'Helps prevent brute-force attacks' + given login_recaptcha_protection_enabled: ->(val) { val } do + requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' + requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' + end optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' @@ -123,7 +132,7 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' - optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" + optional :local_markdown_version, type: Integer, desc: 'Local markdown version, increase this value when any cached markdown should be invalidated' optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' given snowplow_enabled: ->(val) { val } do diff --git a/lib/api/validations/types/comma_separated_to_array.rb b/lib/api/validations/types/comma_separated_to_array.rb new file mode 100644 index 00000000000..b551878abd1 --- /dev/null +++ b/lib/api/validations/types/comma_separated_to_array.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module API + module Validations + module Types + class CommaSeparatedToArray + def self.coerce + lambda do |value| + case value + when String + value.split(',').map(&:strip) + when Array + value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + else + [] + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 52af28ce8ec..a0439089879 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -7,6 +7,14 @@ module Banzai class AbstractReferenceFilter < ReferenceFilter include CrossProjectReference + # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found + # reference (which we replace with placeholder during re-scaping). The + # random number helps ensure it's pretty close to unique. Since it's a + # transitory value (it never gets saved) we can initialize once, and it + # doesn't matter if it changes on a restart. + REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" + REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze + def self.object_class # Implement in child class # Example: MergeRequest @@ -389,6 +397,14 @@ module Banzai def escape_html_entities(text) CGI.escapeHTML(text.to_s) end + + def escape_with_placeholders(text, placeholder_data) + escaped = escape_html_entities(text) + + escaped.gsub(REFERENCE_PLACEHOLDER_PATTERN) do |match| + placeholder_data[$1.to_i] + end + end end end end diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb new file mode 100644 index 00000000000..0a9a52a73a1 --- /dev/null +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Proxy's images/assets to another server. Reduces mixed content warnings + # as well as hiding the customer's IP address when requesting images. + # Copies the original img `src` to `data-canonical-src` then replaces the + # `src` with a new url to the proxy server. + class AssetProxyFilter < HTML::Pipeline::CamoFilter + def initialize(text, context = nil, result = nil) + super + end + + def validate + needs(:asset_proxy, :asset_proxy_secret_key) if asset_proxy_enabled? + end + + def asset_host_whitelisted?(host) + context[:asset_proxy_domain_regexp] ? context[:asset_proxy_domain_regexp].match?(host) : false + end + + def self.transform_context(context) + context[:disable_asset_proxy] = !Gitlab.config.asset_proxy.enabled + + unless context[:disable_asset_proxy] + context[:asset_proxy_enabled] = !context[:disable_asset_proxy] + context[:asset_proxy] = Gitlab.config.asset_proxy.url + context[:asset_proxy_secret_key] = Gitlab.config.asset_proxy.secret_key + context[:asset_proxy_domain_regexp] = Gitlab.config.asset_proxy.domain_regexp + end + + context + end + + # called during an initializer. Caching the values in Gitlab.config significantly increased + # performance, rather than querying Gitlab::CurrentSettings.current_application_settings + # over and over. However, this does mean that the Rails servers need to get restarted + # whenever the application settings are changed + def self.initialize_settings + application_settings = Gitlab::CurrentSettings.current_application_settings + Gitlab.config['asset_proxy'] ||= Settingslogic.new({}) + + if application_settings.respond_to?(:asset_proxy_enabled) + Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled + Gitlab.config.asset_proxy['url'] = application_settings.asset_proxy_url + Gitlab.config.asset_proxy['secret_key'] = application_settings.asset_proxy_secret_key + Gitlab.config.asset_proxy['whitelist'] = application_settings.asset_proxy_whitelist || [Gitlab.config.gitlab.host] + Gitlab.config.asset_proxy['domain_regexp'] = compile_whitelist(Gitlab.config.asset_proxy.whitelist) + else + Gitlab.config.asset_proxy['enabled'] = ::ApplicationSetting.defaults[:asset_proxy_enabled] + end + end + + def self.compile_whitelist(domain_list) + return if domain_list.empty? + + escaped = domain_list.map { |domain| Regexp.escape(domain).gsub('\*', '.*?') } + Regexp.new("^(#{escaped.join('|')})$", Regexp::IGNORECASE) + end + end + end +end diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb index f49c4b403db..02a47556151 100644 --- a/lib/banzai/filter/commit_trailers_filter.rb +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -88,7 +88,8 @@ module Banzai user: user, user_email: email, css_class: 'avatar-inline', - has_tooltip: false + has_tooltip: false, + only_path: false ) link_href = user.nil? ? "mailto:#{email}" : urls.user_url(user) diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 61ee3eac216..fb721fe12b1 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -14,10 +14,10 @@ module Banzai # such as on `mailto:` links. Since we've been using it, do an # initial parse for validity and then use Addressable # for IDN support, etc - uri = uri_strict(node['href'].to_s) + uri = uri_strict(node_src(node)) if uri - node.set_attribute('href', uri.to_s) - addressable_uri = addressable_uri(node['href']) + node.set_attribute(node_src_attribute(node), uri.to_s) + addressable_uri = addressable_uri(node_src(node)) else addressable_uri = nil end @@ -35,6 +35,16 @@ module Banzai private + # if this is a link to a proxied image, then `src` is already the correct + # proxied url, so work with the `data-canonical-src` + def node_src_attribute(node) + node['data-canonical-src'] ? 'data-canonical-src' : 'href' + end + + def node_src(node) + node[node_src_attribute(node)] + end + def uri_strict(href) URI.parse(href) rescue URI::Error @@ -72,7 +82,7 @@ module Banzai return unless uri return unless context[:emailable_links] - unencoded_uri_str = Addressable::URI.unencode(node['href']) + unencoded_uri_str = Addressable::URI.unencode(node_src(node)) if unencoded_uri_str == node.content && idn?(uri) node.content = uri.normalize diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index 01237303c27..ed0a01e6277 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -18,6 +18,9 @@ module Banzai rel: 'noopener noreferrer' ) + # make sure the original non-proxied src carries over to the link + link['data-canonical-src'] = img['data-canonical-src'] if img['data-canonical-src'] + link.children = img.clone img.replace(link) diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 8e2358694d4..9dfd77b1759 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -21,7 +21,8 @@ module Banzai next if !can_read_cross_project? && cross_reference?(issuable) if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) - node.content += " (#{issuable.state})" + state = moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state + node.content += " (#{state})" end end @@ -30,6 +31,10 @@ module Banzai private + def moved_issue?(issuable) + issuable.instance_of?(Issue) && issuable.moved? + end + def issuable_reference?(text, issuable) CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 4892668fc22..a0789b7ca06 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -14,24 +14,24 @@ module Banzai find_labels(parent_object).find(id) end - def self.references_in(text, pattern = Label.reference_pattern) - unescape_html_entities(text).gsub(pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~[:namespace], $~ - end - end - def references_in(text, pattern = Label.reference_pattern) - unescape_html_entities(text).gsub(pattern) do |match| + labels = {} + unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| namespace, project = $~[:namespace], $~[:project] project_path = full_project_path(namespace, project) label = find_label(project_path, $~[:label_id], $~[:label_name]) if label - yield match, label.id, project, namespace, $~ + labels[label.id] = yield match, label.id, project, namespace, $~ + "#{REFERENCE_PLACEHOLDER}#{label.id}" else - escape_html_entities(match) + match end end + + return text if labels.empty? + + escape_with_placeholders(unescaped_html, labels) end def find_label(parent_ref, label_id, label_name) diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 08969753d75..4c47ee4dba1 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -51,15 +51,21 @@ module Banzai # default implementation. return super(text, pattern) if pattern != Milestone.reference_pattern - unescape_html_entities(text).gsub(pattern) do |match| + milestones = {} + unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone - yield match, milestone.id, $~[:project], $~[:namespace], $~ + milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~ + "#{REFERENCE_PLACEHOLDER}#{milestone.id}" else - escape_html_entities(match) + match end end + + return text if milestones.empty? + + escape_with_placeholders(unescaped_html, milestones) end def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 86f18679496..846a7d46aad 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -9,6 +9,7 @@ module Banzai # Context options: # :commit # :group + # :current_user # :project # :project_wiki # :ref @@ -18,6 +19,7 @@ module Banzai def call return doc if context[:system_note] + return doc unless visible_to_user? @uri_types = {} clear_memoization(:linkable_files) @@ -166,6 +168,16 @@ module Banzai Gitlab.config.gitlab.relative_url_root.presence || '/' end + def visible_to_user? + if project + Ability.allowed?(current_user, :download_code, project) + elsif group + Ability.allowed?(current_user, :read_group, group) + else # Objects detached from projects or groups, e.g. Personal Snippets. + true + end + end + def ref context[:ref] || project.default_branch end @@ -178,6 +190,10 @@ module Banzai context[:project] end + def current_user + context[:current_user] + end + def repository @repository ||= project&.repository end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index 0fff104cf91..a278fcfdb47 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -23,6 +23,14 @@ module Banzai "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})" end + if context[:asset_proxy_enabled].present? + src_query.concat( + UploaderHelper::VIDEO_EXT.map do |ext| + "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})" + end + ) + end + "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]" end end @@ -48,6 +56,13 @@ module Banzai target: '_blank', rel: 'noopener noreferrer', title: "Download '#{element['title'] || element['alt']}'") + + # make sure the original non-proxied src carries over + if element['data-canonical-src'] + video['data-canonical-src'] = element['data-canonical-src'] + link['data-canonical-src'] = element['data-canonical-src'] + end + download_paragraph = doc.document.create_element('p') download_paragraph.children = link diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index d25b74b23b2..82b99d3de4a 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -6,12 +6,17 @@ module Banzai def self.filters FilterArray[ Filter::AsciiDocSanitizationFilter, + Filter::AssetProxyFilter, Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::AsciiDocPostProcessingFilter ] end + + def self.transform_context(context) + Filter::AssetProxyFilter.transform_context(context) + end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 2c1006f708a..f419e54c264 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -17,6 +17,7 @@ module Banzai Filter::SpacedLinkFilter, Filter::SanitizationFilter, + Filter::AssetProxyFilter, Filter::SyntaxHighlightFilter, Filter::MathFilter, @@ -60,7 +61,7 @@ module Banzai def self.transform_context(context) context[:only_path] = true unless context.key?(:only_path) - context + Filter::AssetProxyFilter.transform_context(context) end end end diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index ceba082cd4f..c86d5f08ded 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -6,11 +6,16 @@ module Banzai def self.filters @filters ||= FilterArray[ Filter::SanitizationFilter, + Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::SyntaxHighlightFilter ] end + + def self.transform_context(context) + Filter::AssetProxyFilter.transform_context(context) + end end end end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 72374207a8f..9aff6880f56 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -7,6 +7,7 @@ module Banzai @filters ||= FilterArray[ Filter::HtmlEntityFilter, Filter::SanitizationFilter, + Filter::AssetProxyFilter, Filter::EmojiFilter, Filter::AutolinkFilter, @@ -29,6 +30,8 @@ module Banzai end def self.transform_context(context) + context = Filter::AssetProxyFilter.transform_context(context) + super(context).merge( no_sourcepos: true ) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 9ded1aed4e3..656becbffd3 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -6,8 +6,11 @@ class Feature class Gitaly # Server feature flags should use '_' to separate words. SERVER_FEATURE_FLAGS = - [ - # 'get_commit_signatures'.freeze + %w[ + get_commit_signatures + cache_invalidator + inforef_uploadpack_cache + get_all_lfs_pointers_go ].freeze DEFAULT_ON_FLAGS = Set.new([]).freeze diff --git a/lib/gitlab.rb b/lib/gitlab.rb index d9d8dcf7900..e8b938e46b1 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -55,7 +55,7 @@ module Gitlab SUBDOMAIN_REGEX === Gitlab.config.gitlab.url end - def self.dev_env_or_com? + def self.dev_env_org_or_com? Rails.env.development? || org? || com? end diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index fdb06d00548..0e8707af631 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -49,9 +49,9 @@ module Gitlab request_information = { message: 'Action_Rate_Limiter_Request', env: type, - ip: request.ip, + remote_ip: request.ip, request_method: request.request_method, - fullpath: request.fullpath + path: request.fullpath } if current_user diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb new file mode 100644 index 00000000000..286c393005f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# This module represents the default Cycle Analytics stages that are currently provided by CE +# Each method returns a hash that can be used to build a new stage object. +# +# Example: +# +# params = Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_issue_stage +# Analytics::CycleAnalytics::ProjectStage.new(params) +module Gitlab + module Analytics + module CycleAnalytics + module DefaultStages + def self.all + [ + params_for_issue_stage, + params_for_plan_stage, + params_for_code_stage, + params_for_test_stage, + params_for_review_stage, + params_for_staging_stage, + params_for_production_stage + ] + end + + def self.params_for_issue_stage + { + name: 'issue', + custom: false, # this stage won't be customizable, we provide it as it is + relative_position: 1, # when opening the CycleAnalytics page in CE, this stage will be the first item + start_event_identifier: :issue_created, # IssueCreated class is used as start event + end_event_identifier: :issue_stage_end # IssueStageEnd class is used as end event + } + end + + def self.params_for_plan_stage + { + name: 'plan', + custom: false, + relative_position: 2, + start_event_identifier: :plan_stage_start, + end_event_identifier: :issue_first_mentioned_in_commit + } + end + + def self.params_for_code_stage + { + name: 'code', + custom: false, + relative_position: 3, + start_event_identifier: :code_stage_start, + end_event_identifier: :merge_request_created + } + end + + def self.params_for_test_stage + { + name: 'test', + custom: false, + relative_position: 4, + start_event_identifier: :merge_request_last_build_started, + end_event_identifier: :merge_request_last_build_finished + } + end + + def self.params_for_review_stage + { + name: 'review', + custom: false, + relative_position: 5, + start_event_identifier: :merge_request_created, + end_event_identifier: :merge_request_merged + } + end + + def self.params_for_staging_stage + { + name: 'staging', + custom: false, + relative_position: 6, + start_event_identifier: :merge_request_merged, + end_event_identifier: :merge_request_first_deployed_to_production + } + end + + def self.params_for_production_stage + { + name: 'production', + custom: false, + relative_position: 7, + start_event_identifier: :merge_request_merged, + end_event_identifier: :merge_request_first_deployed_to_production + } + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb new file mode 100644 index 00000000000..d21f344f483 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Convention: + # Issue: < 100 + # MergeRequest: >= 100 && < 1000 + # Custom events for default stages: >= 1000 (legacy) + ENUM_MAPPING = { + StageEvents::IssueCreated => 1, + StageEvents::IssueFirstMentionedInCommit => 2, + StageEvents::MergeRequestCreated => 100, + StageEvents::MergeRequestFirstDeployedToProduction => 101, + StageEvents::MergeRequestLastBuildFinished => 102, + StageEvents::MergeRequestLastBuildStarted => 103, + StageEvents::MergeRequestMerged => 104, + StageEvents::CodeStageStart => 1_000, + StageEvents::IssueStageEnd => 1_001, + StageEvents::PlanStageStart => 1_002 + }.freeze + + EVENTS = ENUM_MAPPING.keys.freeze + + # Defines which start_event and end_event pairs are allowed + PAIRING_RULES = { + StageEvents::PlanStageStart => [ + StageEvents::IssueFirstMentionedInCommit + ], + StageEvents::CodeStageStart => [ + StageEvents::MergeRequestCreated + ], + StageEvents::IssueCreated => [ + StageEvents::IssueStageEnd + ], + StageEvents::MergeRequestCreated => [ + StageEvents::MergeRequestMerged + ], + StageEvents::MergeRequestLastBuildStarted => [ + StageEvents::MergeRequestLastBuildFinished + ], + StageEvents::MergeRequestMerged => [ + StageEvents::MergeRequestFirstDeployedToProduction + ] + }.freeze + + def [](identifier) + events.find { |e| e.identifier.to_s.eql?(identifier.to_s) } || raise(KeyError) + end + + # hash for defining ActiveRecord enum: identifier => number + def to_enum + ENUM_MAPPING.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v } + end + + # will be overridden in EE with custom events + def pairing_rules + PAIRING_RULES + end + + # will be overridden in EE with custom events + def events + EVENTS + end + + module_function :[], :to_enum, :pairing_rules, :events + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb new file mode 100644 index 00000000000..ff9c8a79225 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class CodeStageStart < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first mentioned in a commit") + end + + def self.identifier + :code_stage_start + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb new file mode 100644 index 00000000000..a601c9797f8 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueCreated < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue created") + end + + def self.identifier + :issue_created + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb new file mode 100644 index 00000000000..7424043ef7b --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueFirstMentionedInCommit < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first mentioned in a commit") + end + + def self.identifier + :issue_first_mentioned_in_commit + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb new file mode 100644 index 00000000000..ceb229c552f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueStageEnd < SimpleStageEvent + def self.name + PlanStageStart.name + end + + def self.identifier + :issue_stage_end + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb new file mode 100644 index 00000000000..8be00831b4f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestCreated < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request created") + end + + def self.identifier + :merge_request_created + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb new file mode 100644 index 00000000000..6d7a2c023ff --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestFirstDeployedToProduction < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request first deployed to production") + end + + def self.identifier + :merge_request_first_deployed_to_production + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb new file mode 100644 index 00000000000..12d82fe2c62 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestLastBuildFinished < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request last build finish time") + end + + def self.identifier + :merge_request_last_build_finished + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb new file mode 100644 index 00000000000..9e749b0fdfa --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestLastBuildStarted < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request last build start time") + end + + def self.identifier + :merge_request_last_build_started + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb new file mode 100644 index 00000000000..bbfb5d12992 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestMerged < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request merged") + end + + def self.identifier + :merge_request_merged + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb new file mode 100644 index 00000000000..803317d8b55 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class PlanStageStart < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board") + end + + def self.identifier + :plan_stage_start + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb new file mode 100644 index 00000000000..253c489d822 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Represents a simple event that usually refers to one database column and does not require additional user input + class SimpleStageEvent < StageEvent + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb new file mode 100644 index 00000000000..a55eee048c2 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Base class for expressing an event that can be used for a stage. + class StageEvent + def initialize(params) + @params = params + end + + def self.name + raise NotImplementedError + end + + def self.identifier + raise NotImplementedError + end + + def object_type + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb new file mode 100644 index 00000000000..148b6d3310d --- /dev/null +++ b/lib/gitlab/anonymous_session.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + class AnonymousSession + def initialize(remote_ip, session_id: nil) + @remote_ip = remote_ip + @session_id = session_id + end + + def store_session_id_per_ip + Gitlab::Redis::SharedState.with do |redis| + redis.pipelined do + redis.sadd(session_lookup_name, session_id) + redis.expire(session_lookup_name, 24.hours) + end + end + end + + def stored_sessions + Gitlab::Redis::SharedState.with do |redis| + redis.scard(session_lookup_name) + end + end + + def cleanup_session_per_ip_entries + Gitlab::Redis::SharedState.with do |redis| + redis.srem(session_lookup_name, session_id) + end + end + + private + + attr_reader :remote_ip, :session_id + + def session_lookup_name + @session_lookup_name ||= "#{Gitlab::Redis::SharedState::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 82e0c7ceeaa..6769bd95c2b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -46,7 +46,7 @@ module Gitlab user_with_password_for_git(login, password) || Gitlab::Auth::Result.new - rate_limit!(ip, success: result.success?, login: login) + rate_limit!(ip, success: result.success?, login: login) unless skip_rate_limit?(login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -119,6 +119,10 @@ module Gitlab private + def skip_rate_limit?(login:) + ::Ci::Build::CI_REGISTRY_USER == login + end + def authenticate_using_internal_or_ldap_password? Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? end @@ -194,12 +198,10 @@ module Gitlab end.uniq end - # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? - token = - DeployToken.active.find_by(token: password) + token = DeployToken.active.find_by_token(password) return unless token && login return if login != token.username @@ -210,7 +212,6 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end - # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 09d1d79fefc..f121dce4cbb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -77,7 +77,12 @@ module Gitlab end def bypass_two_factor? - false + providers = Gitlab.config.omniauth.allow_bypass_two_factor + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end protected diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index 3fe72f5fd43..820a78b653c 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -13,6 +13,24 @@ module Gitlab @logger = logger end + # Checks if the file is accessible or not + # + # @return [Boolean] + def accessible? + open_authorized_keys_file('r') { true } + rescue Errno::ENOENT, Errno::EACCES + false + end + + # Creates the authorized_keys file if it doesn't exist + # + # @return [Boolean] + def create + open_authorized_keys_file(File::CREAT) { true } + rescue Errno::EACCES + false + end + # Add id and its key to the authorized_keys file # # @param [String] id identifier of key prefixed by `key-` @@ -102,10 +120,14 @@ module Gitlab [] end + def file + @file ||= Gitlab.config.gitlab_shell.authorized_keys_file + end + private def lock(timeout = 10) - File.open("#{authorized_keys_file}.lock", "w+") do |f| + File.open("#{file}.lock", "w+") do |f| f.flock File::LOCK_EX Timeout.timeout(timeout) { yield } ensure @@ -114,7 +136,7 @@ module Gitlab end def open_authorized_keys_file(mode) - File.open(authorized_keys_file, mode, 0o600) do |file| + File.open(file, mode, 0o600) do |file| file.chmod(0o600) yield file end @@ -141,9 +163,5 @@ module Gitlab def strip(key) key.split(/[ ]+/)[0, 2].join(' ') end - - def authorized_keys_file - Gitlab.config.gitlab_shell.authorized_keys_file - end end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 0698136166a..e9c8864123f 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - variables = seed.to_resource.scoped_variables_hash + variables = seed.scoped_variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb new file mode 100644 index 00000000000..89623a809c9 --- /dev/null +++ b/lib/gitlab/ci/build/rules.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules + include ::Gitlab::Utils::StrongMemoize + + Result = Struct.new(:when, :start_in) + + def initialize(rule_hashes, default_when = 'on_success') + @rule_list = Rule.fabricate_list(rule_hashes) + @default_when = default_when + end + + def evaluate(pipeline, build) + if @rule_list.nil? + Result.new(@default_when) + elsif matched_rule = match_rule(pipeline, build) + Result.new( + matched_rule.attributes[:when] || @default_when, + matched_rule.attributes[:start_in] + ) + else + Result.new('never') + end + end + + private + + def match_rule(pipeline, build) + @rule_list.find { |rule| rule.matches?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb new file mode 100644 index 00000000000..8d52158c8d2 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule + attr_accessor :attributes + + def self.fabricate_list(list) + list.map(&method(:new)) if list + end + + def initialize(spec) + @clauses = [] + @attributes = {} + + spec.each do |type, value| + if clause = Clause.fabricate(type, value) + @clauses << clause + else + @attributes.merge!(type => value) + end + end + end + + def matches?(pipeline, build) + @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb new file mode 100644 index 00000000000..ff0baf3348c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause + ## + # Abstract class that defines an interface of a single + # job rule specification. + # + # Used for job's inclusion rules configuration. + # + UnknownClauseError = Class.new(StandardError) + + def self.fabricate(type, value) + type = type.to_s.camelize + + self.const_get(type).new(value) if self.const_defined?(type) + end + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline, seed = nil) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb new file mode 100644 index 00000000000..81d2ee6c24c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::Changes < Rules::Rule::Clause + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true if pipeline.modified_paths.nil? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb new file mode 100644 index 00000000000..18c3b450f95 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::If < Rules::Rule::Clause + def initialize(expression) + @expression = expression + end + + def satisfied_by?(pipeline, seed) + variables = seed.scoped_variables_hash + + ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful? + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 29a52b9da17..6e11c582750 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,8 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[tags script only except type image services + ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze + ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache dependencies needs before_script after_script variables environment coverage retry parallel extends].freeze @@ -19,12 +20,19 @@ module Gitlab REQUIRED_BY_NEEDS = %i[stage].freeze validations do + validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :config, presence: true validates :script, presence: true validates :name, presence: true validates :name, type: Symbol + validates :config, + disallowed_keys: { + in: %i[only except when start_in], + message: 'key may not be used with `rules`' + }, + if: :has_rules? with_options allow_nil: true do validates :tags, array_of_strings: true @@ -32,17 +40,19 @@ module Gitlab validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } - validates :when, - inclusion: { in: %w[on_success on_failure always manual delayed], - message: 'should be on_success, on_failure, ' \ - 'always, manual or delayed' } + validates :when, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } + validates :dependencies, array_of_strings: true validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true + validates :rules, array_of_hashes: true end validates :start_in, duration: { limit: '1 day' }, if: :delayed? - validates :start_in, absence: true, unless: :delayed? + validates :start_in, absence: true, if: -> { has_rules? || !delayed? } validate do next unless dependencies.present? @@ -91,6 +101,9 @@ module Gitlab entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' + entry :rules, Entry::Rules, + description: 'List of evaluable Rules to determine job inclusion.' + entry :variables, Entry::Variables, description: 'Environment variables available for this job.' @@ -112,7 +125,7 @@ module Gitlab :parallel, :needs attributes :script, :tags, :allow_failure, :when, :dependencies, - :needs, :retry, :parallel, :extends, :start_in + :needs, :retry, :parallel, :extends, :start_in, :rules def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -151,6 +164,10 @@ module Gitlab self.when == 'delayed' end + def has_rules? + @config.try(:key?, :rules) + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb new file mode 100644 index 00000000000..65cad0880f5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: Array + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |rule, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule) + .value(rule) + .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb new file mode 100644 index 00000000000..1f2a34ec90e --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules::Rule < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + CLAUSES = %i[if changes].freeze + ALLOWED_KEYS = %i[if changes when start_in].freeze + ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze + + attributes :if, :changes, :when, :start_in + + validations do + validates :config, presence: true + validates :config, type: { with: Hash } + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? + validates :start_in, presence: true, if: :specifies_delay? + validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + + with_options allow_nil: true do + validates :if, expression: true + validates :changes, array_of_strings: true + validates :when, allowed_values: { in: ALLOWED_WHEN } + end + end + + def specifies_delay? + self.when == 'delayed' + end + + def default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb new file mode 100644 index 00000000000..31c218bf954 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class JobActivity < Chain::Base + def perform! + # to be overridden in EE + end + + def break? + false # to be overridden in EE + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 942e4e55323..f7b0720d4a9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -11,8 +11,9 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return false unless regexp - regexp.scan(text.to_s).any? + regexp.scan(text.to_s).present? end def self.build(_value, behind, ahead) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 831c27fa0ea..02479ed28a4 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -11,8 +11,9 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return true unless regexp - regexp.scan(text.to_s).none? + regexp.scan(text.to_s).empty? end def self.build(_value, behind, ahead) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 7ec03d132c0..1066331062b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,7 +7,7 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - delegate :dig, to: :@attributes + delegate :dig, to: :@seed_attributes # When the `ci_dag_limit_needs` is enabled it uses the lower limit LOW_NEEDS_LIMIT = 5 @@ -15,14 +15,20 @@ module Gitlab def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline - @attributes = attributes + @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) + @using_rules = attributes.key?(:rules) + @using_only = attributes.key?(:only) + @using_except = attributes.key?(:except) + @only = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:only)) @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) + @rules = Gitlab::Ci::Build::Rules + .new(attributes.delete(:rules)) end def name @@ -31,8 +37,13 @@ module Gitlab def included? strong_memoize(:inclusion) do - all_of_only? && - none_of_except? + if @using_rules + included_by_rules? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end end end @@ -45,19 +56,13 @@ module Gitlab end def attributes - @attributes.merge( - pipeline: @pipeline, - project: @pipeline.project, - user: @pipeline.user, - ref: @pipeline.ref, - tag: @pipeline.tag, - trigger_request: @pipeline.legacy_trigger, - protected: @pipeline.protected_ref? - ) + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) end def bridge? - attributes_hash = @attributes.to_h + attributes_hash = @seed_attributes.to_h attributes_hash.dig(:options, :trigger).present? || (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) && attributes_hash.dig(:options, :bridge_needs, :pipeline).present?) @@ -73,6 +78,18 @@ module Gitlab end end + def scoped_variables_hash + strong_memoize(:scoped_variables_hash) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + ::Ci::Build.new( + @seed_attributes.merge(pipeline_attributes) + ).scoped_variables_hash + end + end + private def all_of_only? @@ -109,6 +126,28 @@ module Gitlab HARD_NEEDS_LIMIT end end + + def pipeline_attributes + { + pipeline: @pipeline, + project: @pipeline.project, + user: @pipeline.user, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + } + end + + def included_by_rules? + rules_attributes[:when] != 'never' + end + + def rules_attributes + strong_memoize(:rules_attributes) do + @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {} + end + end end end end diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 4190de73e1f..90278122361 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -46,11 +46,14 @@ sast: SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ + SAST_JAVA_VERSION \ ANT_HOME \ ANT_PATH \ GRADLE_PATH \ JAVA_OPTS \ JAVA_PATH \ + JAVA_8_VERSION \ + JAVA_11_VERSION \ MAVEN_CLI_OPTS \ MAVEN_PATH \ MAVEN_REPO_PATH \ diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 0289e675c6b..374f929878e 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -20,8 +20,10 @@ module Gitlab present_keys = value.try(:keys).to_a & options[:in] if present_keys.any? - record.errors.add(attribute, "contains disallowed keys: " + - present_keys.join(', ')) + message = options[:message] || "contains disallowed keys" + message += ": #{present_keys.join(', ')}" + + record.errors.add(attribute, message) end end end @@ -65,6 +67,16 @@ module Gitlab end end + class ArrayOfHashesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all? + record.errors.add(attribute, 'should be an array of hashes') + end + end + end + class ArrayOrStringValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless value.is_a?(Array) || value.is_a?(String) @@ -231,6 +243,14 @@ module Gitlab end end + class ExpressionValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid? + record.errors.add(attribute, 'Invalid expression syntax') + end + end + end + class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) return unless value.is_a?(Array) diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 6d5fc4219fb..2f4ae010e74 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -46,7 +46,10 @@ module Gitlab if thread thread.wakeup if thread.alive? - thread.join unless Thread.current == thread + begin + thread.join unless Thread.current == thread + rescue Exception # rubocop:disable Lint/RescueException + end @thread = nil end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 332ca8bf9b8..5424298723e 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -126,6 +126,7 @@ module Gitlab %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, + %r{\A\.rubocop(_todo)?\.yml\z} => :backend, %r{\A(ee/)?qa/} => :qa, diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 74cbcc11255..2789706aa3b 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -41,7 +41,7 @@ module Gitlab when :test area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1] - area && labels.any?(area) if kind == :reviewer + area && labels.any?("devops::#{area.downcase}") if kind == :reviewer else capabilities(project).include?("#{kind} #{category}") end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 37fadb47736..75d9a2d55b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -129,8 +129,6 @@ module Gitlab SAMPLE_DATA end - private - def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cbdff0ab060..6ecd506d55b 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -13,6 +13,10 @@ module Gitlab # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze + # The maximum number of characters for text fields, to avoid DoS attacks via parsing huge text fields + # https://gitlab.com/gitlab-org/gitlab-ce/issues/61974 + MAX_TEXT_SIZE_LIMIT = 1_000_000 + # Minimum schema version from which migrations are supported # Migrations before this version may have been removed MIN_SCHEMA_VERSION = 20190506135400 @@ -195,13 +199,14 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. - def self.create_connection_pool(pool_size, host = nil) + def self.create_connection_pool(pool_size, host = nil, port = nil) # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb env = Rails.env original_config = ActiveRecord::Base.configurations env_config = original_config[env].merge('pool' => pool_size) env_config['host'] = host if host + env_config['port'] = port if port config = original_config.merge(env => env_config) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 9bba4f6ce1e..57a413f8e04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -470,7 +470,7 @@ module Gitlab # We set the default value _after_ adding the column so we don't end up # updating any existing data with the default value. This isn't # necessary since we copy over old values further down. - change_column_default(table, new, old_col.default) if old_col.default + change_column_default(table, new, old_col.default) unless old_col.default.nil? install_rename_triggers(table, old, new) @@ -482,6 +482,16 @@ module Gitlab copy_foreign_keys(table, old, new) end + def undo_rename_column_concurrently(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + check_trigger_permissions!(table) + + remove_rename_triggers_for_postgresql(table, trigger_name) + + remove_column(table, new) + end + # Installs triggers in a table that keep a new column in sync with an old # one. # @@ -547,6 +557,35 @@ module Gitlab remove_column(table, old) end + def undo_cleanup_concurrent_column_rename(table, old, new, type: nil) + if transaction_open? + raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction' + end + + check_trigger_permissions!(table) + + new_column = column_for(table, new) + + add_column(table, old, type || new_column.type, + limit: new_column.limit, + precision: new_column.precision, + scale: new_column.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, old, new_column.default) unless new_column.default.nil? + + install_rename_triggers(table, old, new) + + update_column_in_batches(table, old, Arel::Table.new(table)[new]) + + change_column_null(table, old, false) unless new_column.null + + copy_indexes(table, new, old) + copy_foreign_keys(table, new, old) + end + # Changes the column type of a table using a background migration. # # Because this method uses a background migration it's more suitable for @@ -747,6 +786,11 @@ module Gitlab EOF execute <<-EOF.strip_heredoc + DROP TRIGGER IF EXISTS #{trigger} + ON #{table} + EOF + + execute <<-EOF.strip_heredoc CREATE TRIGGER #{trigger} BEFORE INSERT OR UPDATE ON #{table} diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb new file mode 100644 index 00000000000..3a170e8b5f8 --- /dev/null +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module SelfMonitoring + module Project + class CreateService < ::BaseService + include Stepable + + STEPS_ALLOWED_TO_FAIL = [ + :validate_application_settings, :validate_project_created, :validate_admins + ].freeze + + VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL + PROJECT_NAME = 'GitLab Instance Administration' + + steps :validate_application_settings, + :validate_project_created, + :validate_admins, + :create_group, + :create_project, + :save_project_id, + :add_group_members, + :add_to_whitelist, + :add_prometheus_manual_configuration + + def initialize + super(nil) + end + + def execute! + result = execute_steps + + if result[:status] == :success + result + elsif STEPS_ALLOWED_TO_FAIL.include?(result[:failed_step]) + success + else + raise StandardError, result[:message] + end + end + + private + + def validate_application_settings + return success if application_settings + + log_error(_('No application_settings found')) + error(_('No application_settings found')) + end + + def validate_project_created + return success unless project_created? + + log_error(_('Project already created')) + error(_('Project already created')) + end + + def validate_admins + unless instance_admins.any? + log_error(_('No active admin user found')) + return error(_('No active admin user found')) + end + + success + end + + def create_group + if project_created? + log_info(_('Instance administrators group already exists')) + @group = application_settings.instance_administration_project.owner + return success(group: @group) + end + + @group = ::Groups::CreateService.new(group_owner, create_group_params).execute + + if @group.persisted? + success(group: @group) + else + error(_('Could not create group')) + end + end + + def create_project + if project_created? + log_info(_('Instance administration project already exists')) + @project = application_settings.instance_administration_project + return success(project: project) + end + + @project = ::Projects::CreateService.new(group_owner, create_project_params).execute + + if project.persisted? + success(project: project) + else + log_error(_("Could not create instance administration project. Errors: %{errors}") % { errors: project.errors.full_messages }) + error(_('Could not create project')) + end + end + + def save_project_id + return success if project_created? + + result = application_settings.update(instance_administration_project_id: @project.id) + + if result + success + else + log_error(_("Could not save instance administration project ID, errors: %{errors}") % { errors: application_settings.errors.full_messages }) + error(_('Could not save project ID')) + end + end + + def add_group_members + members = @group.add_users(members_to_add, Gitlab::Access::MAINTAINER) + errors = members.flat_map { |member| member.errors.full_messages } + + if errors.any? + log_error(_('Could not add admins as members to self-monitoring project. Errors: %{errors}') % { errors: errors }) + error(_('Could not add admins as members')) + else + success + end + end + + def add_to_whitelist + return success unless prometheus_enabled? + return success unless prometheus_listen_address.present? + + uri = parse_url(internal_prometheus_listen_address_uri) + return error(_('Prometheus listen_address is not a valid URI')) unless uri + + application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host]) + result = application_settings.save + + if result + # Expire the Gitlab::CurrentSettings cache after updating the whitelist. + # This happens automatically in an after_commit hook, but in migrations, + # the after_commit hook only runs at the end of the migration. + Gitlab::CurrentSettings.expire_current_application_settings + success + else + log_error(_("Could not add prometheus URL to whitelist, errors: %{errors}") % { errors: application_settings.errors.full_messages }) + error(_('Could not add prometheus URL to whitelist')) + end + end + + def add_prometheus_manual_configuration + return success unless prometheus_enabled? + return success unless prometheus_listen_address.present? + + service = project.find_or_initialize_service('prometheus') + + unless service.update(prometheus_service_attributes) + log_error(_('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}') % { errors: service.errors.full_messages }) + return error(_('Could not save prometheus manual configuration')) + end + + success + end + + def application_settings + @application_settings ||= ApplicationSetting.current_without_cache + end + + def project_created? + application_settings.instance_administration_project.present? + end + + def parse_url(uri_string) + Addressable::URI.parse(uri_string) + rescue Addressable::URI::InvalidURIError, TypeError + end + + def prometheus_enabled? + Gitlab.config.prometheus.enable if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + log_error(_('prometheus.enable is not present in gitlab.yml')) + + false + end + + def prometheus_listen_address + Gitlab.config.prometheus.listen_address if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + log_error(_('prometheus.listen_address is not present in gitlab.yml')) + + nil + end + + def instance_admins + @instance_admins ||= User.admins.active + end + + def group_owner + instance_admins.first + end + + def members_to_add + # Exclude admins who are already members of group because + # `@group.add_users(users)` returns an error if the users parameter contains + # users who are already members of the group. + instance_admins - @group.members.collect(&:user) + end + + def create_group_params + { + name: 'GitLab Instance Administrators', + path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}", + visibility_level: VISIBILITY_LEVEL + } + end + + def docs_path + Rails.application.routes.url_helpers.help_page_path( + 'administration/monitoring/gitlab_instance_administration_project/index' + ) + end + + def create_project_params + { + initialize_with_readme: true, + visibility_level: VISIBILITY_LEVEL, + name: PROJECT_NAME, + description: "This project is automatically generated and will be used to help monitor this GitLab instance. [More information](#{docs_path})", + namespace_id: @group.id + } + end + + def internal_prometheus_listen_address_uri + if prometheus_listen_address.starts_with?('http') + prometheus_listen_address + else + 'http://' + prometheus_listen_address + end + end + + def prometheus_service_attributes + { + api_url: internal_prometheus_listen_address_uri, + manual_configuration: true, + active: true + } + end + end + end + end + end +end diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb new file mode 100644 index 00000000000..e48041d9218 --- /dev/null +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class SmimeSignatureInterceptor + # Sign emails with SMIME if enabled + class << self + def delivering_email(message) + signed_message = Gitlab::Email::Smime::Signer.sign( + cert: certificate.cert, + key: certificate.key, + data: message.encoded) + signed_email = Mail.new(signed_message) + + overwrite_body(message, signed_email) + overwrite_headers(message, signed_email) + end + + private + + def certificate + @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path) + end + + def key_path + Gitlab.config.gitlab.email_smime.key_file + end + + def cert_path + Gitlab.config.gitlab.email_smime.cert_file + end + + def overwrite_body(message, signed_email) + # since this is a multipart email, assignment to nil is important, + # otherwise Message#body will add a new mail part + message.body = nil + message.body = signed_email.body.encoded + end + + def overwrite_headers(message, signed_email) + message.content_disposition = signed_email.content_disposition + message.content_transfer_encoding = signed_email.content_transfer_encoding + message.content_type = signed_email.content_type + end + end + end + end + end +end diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb new file mode 100644 index 00000000000..b331c4ca19c --- /dev/null +++ b/lib/gitlab/email/smime/certificate.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Smime + class Certificate + include OpenSSL + + attr_reader :key, :cert + + def key_string + @key.to_s + end + + def cert_string + @cert.to_pem + end + + def self.from_strings(key_string, cert_string) + key = PKey::RSA.new(key_string) + cert = X509::Certificate.new(cert_string) + new(key, cert) + end + + def self.from_files(key_path, cert_path) + from_strings(File.read(key_path), File.read(cert_path)) + end + + def initialize(key, cert) + @key = key + @cert = cert + end + end + end + end +end diff --git a/lib/gitlab/email/smime/signer.rb b/lib/gitlab/email/smime/signer.rb new file mode 100644 index 00000000000..2fa83014003 --- /dev/null +++ b/lib/gitlab/email/smime/signer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'openssl' + +module Gitlab + module Email + module Smime + # Tooling for signing and verifying data with SMIME + class Signer + include OpenSSL + + def self.sign(cert:, key:, data:) + signed_data = PKCS7.sign(cert, key, data, nil, PKCS7::DETACHED) + PKCS7.write_smime(signed_data) + end + + # return nil if data cannot be verified, otherwise the signed content data + def self.verify_signature(cert:, ca_cert: nil, signed_data:) + store = X509::Store.new + store.set_default_paths + store.add_cert(ca_cert) if ca_cert + + signed_smime = PKCS7.read_smime(signed_data) + signed_smime if signed_smime.verify([cert], store) + end + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 3c71031a8d9..841f9de8d4a 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -20,7 +20,7 @@ module Gitlab path: repo.path, namespace: namespace, creator: current_user, - visibility_level: Gitlab::VisibilityLevel::INTERNAL, + visibility_level: Gitlab::VisibilityLevel::PRIVATE, import_type: 'fogbugz', import_source: repo.name, import_url: Project::UNKNOWN_IMPORT_URL, diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e6cbfb00f60..d65c0d3e78d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -50,7 +50,7 @@ module Gitlab def self.interceptors return [] unless Labkit::Tracing.enabled? - [Labkit::Tracing::GRPCInterceptor.instance] + [Labkit::Tracing::GRPC::ClientInterceptor.instance] end private_class_method :interceptors @@ -406,7 +406,8 @@ module Gitlab def self.filesystem_id(storage) response = Gitlab::GitalyClient::ServerService.new(storage).info storage_status = response.storage_statuses.find { |status| status.storage_name == storage } - storage_status.filesystem_id + + storage_status&.filesystem_id end def self.filesystem_id_from_disk(storage) diff --git a/lib/gitlab/grape_logging/loggers/client_env_logger.rb b/lib/gitlab/grape_logging/loggers/client_env_logger.rb new file mode 100644 index 00000000000..3acc6f6a2ef --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/client_env_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This is a fork of +# https://github.com/aserafin/grape_logging/blob/master/lib/grape_logging/loggers/client_env.rb +# to use remote_ip instead of ip. +module Gitlab + module GrapeLogging + module Loggers + class ClientEnvLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { remote_ip: request.env["HTTP_X_FORWARDED_FOR"] || request.env["REMOTE_ADDR"], ua: request.env["HTTP_USER_AGENT"] } + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb new file mode 100644 index 00000000000..a0312366d66 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchRootStorageStatisticsLoader + attr_reader :namespace_id + + def initialize(namespace_id) + @namespace_id = namespace_id + end + + def find + BatchLoader.for(namespace_id).batch do |namespace_ids, loader| + Namespace::RootStorageStatistics.for_namespace_ids(namespace_ids).each do |statistics| + loader.call(statistics.namespace_id, statistics) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index ab03c40c22d..941a4f434a1 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -23,7 +23,9 @@ module Gitlab end presenter = presented_in.presenter_class.new(object, **context.to_h) - wrapped = presented_type.class.new(presenter, context) + + # we have to use the new `authorized_new` method, as `new` is protected + wrapped = presented_type.class.authorized_new(presenter, context) old_resolver.call(wrapped, args, context) end diff --git a/lib/gitlab/internal_post_receive/response.rb b/lib/gitlab/internal_post_receive/response.rb new file mode 100644 index 00000000000..7e7ec2aa45c --- /dev/null +++ b/lib/gitlab/internal_post_receive/response.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module InternalPostReceive + class Response + attr_accessor :reference_counter_decreased + attr_reader :messages + + Message = Struct.new(:message, :type) do + def self.basic(text) + new(text, :basic) + end + + def self.alert(text) + new(text, :alert) + end + end + + def initialize + @messages = [] + @reference_counter_decreased = false + end + + def add_merge_request_urls(urls_data) + urls_data.each do |url_data| + add_merge_request_url(url_data) + end + end + + def add_merge_request_url(url_data) + message = if url_data[:new_merge_request] + "To create a merge request for #{url_data[:branch_name]}, visit:" + else + "View merge request for #{url_data[:branch_name]}:" + end + + message += "\n #{url_data[:url]}" + + add_basic_message(message) + end + + def add_basic_message(text) + @messages << Message.basic(text) if text.present? + end + + def add_alert_message(text) + @messages.unshift(Message.alert(text)) if text.present? + end + end + end +end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb new file mode 100644 index 00000000000..11a33a7b358 --- /dev/null +++ b/lib/gitlab/jira/http_client.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Jira + # Gitlab JIRA HTTP client to be used with jira-ruby gem, this subclasses JIRA::HTTPClient. + # Uses Gitlab::HTTP to make requests to JIRA REST API. + # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.4.0/lib/jira/http_client.rb + class HttpClient < JIRA::HttpClient + extend ::Gitlab::Utils::Override + + override :request + def request(*args) + result = make_request(*args) + + raise JIRA::HTTPError.new(result) unless result.response.is_a?(Net::HTTPSuccess) + + result + end + + override :make_cookie_auth_request + def make_cookie_auth_request + body = { + username: @options.delete(:username), + password: @options.delete(:password) + }.to_json + + make_request(:post, @options[:context_path] + '/rest/auth/1/session', body, { 'Content-Type' => 'application/json' }) + end + + override :make_request + def make_request(http_method, path, body = '', headers = {}) + request_params = { headers: headers } + request_params[:body] = body if body.present? + request_params[:headers][:Cookie] = get_cookies if options[:use_cookies] + request_params[:timeout] = options[:read_timeout] if options[:read_timeout] + request_params[:base_uri] = uri.to_s + request_params.merge!(auth_params) + + result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend + @authenticated = result.response.is_a?(Net::HTTPOK) + store_cookies(result) if options[:use_cookies] + + result + end + + def auth_params + return {} unless @options[:username] && @options[:password] + + { + basic_auth: { + username: @options[:username], + password: @options[:password] + } + } + end + + private + + def get_cookies + cookie_array = @cookies.values.map { |cookie| "#{cookie.name}=#{cookie.value[0]}" } + cookie_array += Array(@options[:additional_cookies]) if @options.key?(:additional_cookies) + cookie_array.join('; ') if cookie_array.any? + end + end + end +end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index 0354c710a3f..03a2f62cbd9 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -3,8 +3,8 @@ module Gitlab module MarkdownCache # Increment this number every time the renderer changes its output + CACHE_COMMONMARK_VERSION = 17 CACHE_COMMONMARK_VERSION_START = 10 - CACHE_COMMONMARK_VERSION = 16 BaseError = Class.new(StandardError) UnsupportedClassError = Class.new(BaseError) diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index f96466b2b00..d9c28ff1181 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -132,7 +132,7 @@ module Gitlab NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze - FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze + FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}}.freeze def root_namespace_route_regex @root_namespace_route_regex ||= begin diff --git a/lib/gitlab/performance_bar/with_top_level_warnings.rb b/lib/gitlab/performance_bar/with_top_level_warnings.rb new file mode 100644 index 00000000000..fb5c5c5959d --- /dev/null +++ b/lib/gitlab/performance_bar/with_top_level_warnings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module PerformanceBar + module WithTopLevelWarnings + def results + results = super + + results.merge(has_warnings: has_warnings?(results)) + end + + def has_warnings?(results) + results[:data].any? do |_, value| + value[:warnings].present? + end + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index ec7671f9a8b..425c30d67fe 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -97,7 +97,7 @@ module Gitlab attr_reader :load_times_by_model, :private_token def debug(message, *) - message.gsub!(private_token, FILTERED_STRING) if private_token + message = message.gsub(private_token, FILTERED_STRING) if private_token _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/) diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 2f78ea05cf0..0fda056a4fe 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -17,8 +17,9 @@ module Gitlab return unless content all_names.each do |a_name| - content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) + content = content.gsub(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) end + content end end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 772d743c9b0..f3cbe1db901 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -3,7 +3,7 @@ module Gitlab module Recaptcha def self.load_configurations! - if Gitlab::CurrentSettings.recaptcha_enabled + if Gitlab::CurrentSettings.recaptcha_enabled || enabled_on_login? ::Recaptcha.configure do |config| config.site_key = Gitlab::CurrentSettings.recaptcha_site_key config.secret_key = Gitlab::CurrentSettings.recaptcha_private_key @@ -16,5 +16,9 @@ module Gitlab def self.enabled? Gitlab::CurrentSettings.recaptcha_enabled end + + def self.enabled_on_login? + Gitlab::CurrentSettings.login_recaptcha_protection_enabled + end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 9066606ca21..270a19e780c 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -9,6 +9,7 @@ module Gitlab SESSION_NAMESPACE = 'session:gitlab'.freeze USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index bb4e4ce7bbc..2f3d14ecebd 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -53,15 +53,18 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def batch_clean(start_id: nil, stop_id: nil, dry_run: true, sleep_time: nil) + def batch_clean(start_id: nil, stop_id: nil, dry_run: true, sleep_time: nil, uploader: nil, since: nil) relation = Upload.where('lower(path) like ? or lower(path) like ? or lower(path) like ?', '%.jpg', '%.jpeg', '%.tiff') + relation = relation.where(uploader: uploader) if uploader + relation = relation.where('created_at > ?', since) if since logger.info "running in dry run mode, no images will be rewritten" if dry_run find_params = { start: start_id.present? ? start_id.to_i : nil, - finish: stop_id.present? ? stop_id.to_i : Upload.last&.id + finish: stop_id.present? ? stop_id.to_i : Upload.last&.id, + batch_size: 1000 } relation.find_each(find_params) do |upload| diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 764db14d720..005cb3112b8 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -39,9 +39,14 @@ module Gitlab # development and test. If you need development and test to behave # just the same as production you can use this instead of # track_exception. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? - extra[:issue_url] = issue_url if issue_url + extra = build_extra_data(exception, issue_url, extra) context # Make sure we've set everything we know in the context Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -58,5 +63,15 @@ module Gitlab locale: I18n.locale } end + + def self.build_extra_data(exception, issue_url, extra) + exception.try(:sentry_extra_data)&.tap do |data| + extra.merge!(data) if data.is_a?(Hash) + end + + extra.merge({ issue_url: issue_url }.compact) + end + + private_class_method :build_extra_data end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0fa17b3f559..9e813968093 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -165,16 +165,7 @@ module Gitlab def add_key(key_id, key_content) return unless self.authorized_keys_enabled? - if shell_out_for_gitlab_keys? - gitlab_shell_fast_execute([ - gitlab_shell_keys_path, - 'add-key', - key_id, - strip_key(key_content) - ]) - else - gitlab_authorized_keys.add_key(key_id, key_content) - end + gitlab_authorized_keys.add_key(key_id, key_content) end # Batch-add keys to authorized_keys @@ -184,19 +175,7 @@ module Gitlab def batch_add_keys(keys) return unless self.authorized_keys_enabled? - if shell_out_for_gitlab_keys? - begin - IO.popen("#{gitlab_shell_keys_path} batch-add-keys", 'w') do |io| - add_keys_to_io(keys, io) - end - - $?.success? - rescue Error - false - end - else - gitlab_authorized_keys.batch_add_keys(keys) - end + gitlab_authorized_keys.batch_add_keys(keys) end # Remove ssh key from authorized_keys @@ -207,11 +186,7 @@ module Gitlab def remove_key(id, _ = nil) return unless self.authorized_keys_enabled? - if shell_out_for_gitlab_keys? - gitlab_shell_fast_execute([gitlab_shell_keys_path, 'rm-key', id]) - else - gitlab_authorized_keys.rm_key(id) - end + gitlab_authorized_keys.rm_key(id) end # Remove all ssh keys from gitlab shell @@ -222,11 +197,7 @@ module Gitlab def remove_all_keys return unless self.authorized_keys_enabled? - if shell_out_for_gitlab_keys? - gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) - else - gitlab_authorized_keys.clear - end + gitlab_authorized_keys.clear end # Remove ssh keys from gitlab shell that are not in the DB @@ -341,14 +312,6 @@ module Gitlab File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name) end - def gitlab_shell_projects_path - File.join(gitlab_shell_path, 'bin', 'gitlab-projects') - end - - def gitlab_shell_keys_path - File.join(gitlab_shell_path, 'bin', 'gitlab-keys') - end - def authorized_keys_enabled? # Return true if nil to ensure the authorized_keys methods work while # fixing the authorized_keys file during migration. @@ -359,35 +322,6 @@ module Gitlab private - def shell_out_for_gitlab_keys? - Gitlab.config.gitlab_shell.authorized_keys_file.blank? - end - - def gitlab_shell_fast_execute(cmd) - output, status = gitlab_shell_fast_execute_helper(cmd) - - return true if status.zero? - - Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") # rubocop:disable Gitlab/RailsLogger - false - end - - def gitlab_shell_fast_execute_raise_error(cmd, vars = {}) - output, status = gitlab_shell_fast_execute_helper(cmd, vars) - - raise Error, output unless status.zero? - - true - end - - def gitlab_shell_fast_execute_helper(cmd, vars = {}) - vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)) - - # Don't pass along the entire parent environment to prevent gitlab-shell - # from wasting I/O by searching through GEM_PATH - Bundler.with_original_env { Popen.popen(cmd, nil, vars) } - end - def git_timeout Gitlab.config.gitlab_shell.git_timeout end @@ -407,16 +341,8 @@ module Gitlab def batch_read_key_ids(batch_size: 100, &block) return unless self.authorized_keys_enabled? - if shell_out_for_gitlab_keys? - IO.popen("#{gitlab_shell_keys_path} list-key-ids") do |key_id_stream| - key_id_stream.lazy.each_slice(batch_size) do |lines| - yield(lines.map { |l| l.chomp.to_i }) - end - end - else - gitlab_authorized_keys.list_key_ids.lazy.each_slice(batch_size) do |key_ids| - yield(key_ids) - end + gitlab_authorized_keys.list_key_ids.lazy.each_slice(batch_size) do |key_ids| + yield(key_ids) end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 60782306ade..48b1524f9c7 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -8,16 +8,16 @@ module Gitlab MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) - started_at = current_time + started_time = get_time base_payload = parse_job(job) - Sidekiq.logger.info log_job_start(started_at, base_payload) + Sidekiq.logger.info log_job_start(base_payload) yield - Sidekiq.logger.info log_job_done(job, started_at, base_payload) + Sidekiq.logger.info log_job_done(job, started_time, base_payload) rescue => job_exception - Sidekiq.logger.warn log_job_done(job, started_at, base_payload, job_exception) + Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) raise end @@ -32,7 +32,7 @@ module Gitlab output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) end - def log_job_start(started_at, payload) + def log_job_start(payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' @@ -45,11 +45,12 @@ module Gitlab payload end - def log_job_done(job, started_at, payload, job_exception = nil) + def log_job_done(job, started_time, payload, job_exception = nil) payload = payload.dup add_instrumentation_keys!(job, payload) - payload['duration'] = elapsed(started_at) - payload['completed_at'] = Time.now.utc + + elapsed_time = elapsed(started_time) + add_time_keys!(elapsed_time, payload) message = base_message(payload) @@ -69,6 +70,14 @@ module Gitlab payload end + def add_time_keys!(time, payload) + payload['duration'] = time[:duration].round(3) + payload['system_s'] = time[:stime].round(3) + payload['user_s'] = time[:utime].round(3) + payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0 + payload['completed_at'] = Time.now.utc + end + def parse_job(job) job = job.dup @@ -93,8 +102,25 @@ module Gitlab (Time.now.utc - start).to_f.round(3) end - def elapsed(start) - (current_time - start).round(3) + def elapsed(t0) + t1 = get_time + { + duration: t1[:now] - t0[:now], + stime: t1[:times][:stime] - t0[:times][:stime], + utime: t1[:times][:utime] - t0[:times][:utime], + ctime: ctime(t1[:times]) - ctime(t0[:times]) + } + end + + def get_time + { + now: current_time, + times: Process.times + } + end + + def ctime(times) + times[:cstime] + times[:cutime] end def current_time diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index b06ffa9c121..368f37a5d8c 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -3,6 +3,10 @@ module Gitlab module SidekiqMiddleware class Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + def initialize @metrics = init_metrics end @@ -31,7 +35,7 @@ module Gitlab def init_metrics { - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job'), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb new file mode 100644 index 00000000000..53a6132edac --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Monitor + def call(worker, job, queue) + Gitlab::SidekiqMonitor.instance.within_job(job['jid'], queue) do + yield + end + rescue Gitlab::SidekiqMonitor::CancelledError + # push job to DeadSet + payload = ::Sidekiq.dump_json(job) + ::Sidekiq::DeadSet.new.kill(payload, notify_failure: false) + + # ignore retries + raise ::Sidekiq::JobRetry::Skip + end + end + end +end diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb new file mode 100644 index 00000000000..9842f1f53f7 --- /dev/null +++ b/lib/gitlab/sidekiq_monitor.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +module Gitlab + class SidekiqMonitor < Daemon + include ::Gitlab::Utils::StrongMemoize + + NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze + CANCEL_DEADLINE = 24.hours.seconds + RECONNECT_TIME = 3.seconds + + # We use exception derived from `Exception` + # to consider this as an very low-level exception + # that should not be caught by application + CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException + + attr_reader :jobs_thread + attr_reader :jobs_mutex + + def initialize + super + + @jobs_thread = {} + @jobs_mutex = Mutex.new + end + + def within_job(jid, queue) + jobs_mutex.synchronize do + jobs_thread[jid] = Thread.current + end + + if cancelled?(jid) + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'run', + queue: queue, + jid: jid, + canceled: true + ) + raise CancelledError + end + + yield + ensure + jobs_mutex.synchronize do + jobs_thread.delete(jid) + end + end + + def self.cancel_job(jid) + payload = { + action: 'cancel', + jid: jid + }.to_json + + ::Gitlab::Redis::SharedState.with do |redis| + redis.setex(cancel_job_key(jid), CANCEL_DEADLINE, 1) + redis.publish(NOTIFICATION_CHANNEL, payload) + end + end + + private + + def start_working + Sidekiq.logger.info( + class: self.class.to_s, + action: 'start', + message: 'Starting Monitor Daemon' + ) + + while enabled? + process_messages + sleep(RECONNECT_TIME) + end + + ensure + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'stop', + message: 'Stopping Monitor Daemon' + ) + end + + def stop_working + thread.raise(Interrupt) if thread.alive? + end + + def process_messages + ::Gitlab::Redis::SharedState.with do |redis| + redis.subscribe(NOTIFICATION_CHANNEL) do |on| + on.message do |channel, message| + process_message(message) + end + end + end + rescue Exception => e # rubocop:disable Lint/RescueException + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'exception', + message: e.message + ) + + # we re-raise system exceptions + raise e unless e.is_a?(StandardError) + end + + def process_message(message) + Sidekiq.logger.info( + class: self.class.to_s, + channel: NOTIFICATION_CHANNEL, + message: 'Received payload on channel', + payload: message + ) + + message = safe_parse(message) + return unless message + + case message['action'] + when 'cancel' + process_job_cancel(message['jid']) + else + # unknown message + end + end + + def safe_parse(message) + JSON.parse(message) + rescue JSON::ParserError + end + + def process_job_cancel(jid) + return unless jid + + # try to find thread without lock + return unless find_thread_unsafe(jid) + + Thread.new do + # try to find a thread, but with guaranteed + # that handle for thread corresponds to actually + # running job + find_thread_with_lock(jid) do |thread| + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'cancel', + message: 'Canceling thread with CancelledError', + jid: jid, + thread_id: thread.object_id + ) + + thread&.raise(CancelledError) + end + end + end + + # This method needs to be thread-safe + # This is why it passes thread in block, + # to ensure that we do process this thread + def find_thread_unsafe(jid) + jobs_thread[jid] + end + + def find_thread_with_lock(jid) + # don't try to lock if we cannot find the thread + return unless find_thread_unsafe(jid) + + jobs_mutex.synchronize do + find_thread_unsafe(jid).tap do |thread| + yield(thread) if thread + end + end + end + + def cancelled?(jid) + ::Gitlab::Redis::SharedState.with do |redis| + redis.exists(self.class.cancel_job_key(jid)) + end + end + + def self.cancel_job_key(jid) + "sidekiq:cancel:#{jid}" + end + end +end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 7c963fcf38a..905e0ec5cc1 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -9,6 +9,7 @@ module Gitlab Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, Gitlab::SlashCommands::IssueMove, + Gitlab::SlashCommands::IssueClose, Gitlab::SlashCommands::Deploy, Gitlab::SlashCommands::Run ] diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb new file mode 100644 index 00000000000..5fcc86e91c4 --- /dev/null +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + class IssueClose < IssueCommand + def self.match(text) + /\Aissue\s+close\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) + end + + def self.help_message + "issue close <id>" + end + + def self.allowed?(project, user) + can?(user, :update_issue, project) + end + + def execute(match) + issue = find_by_iid(match[:iid]) + + return not_found unless issue + return presenter(issue).already_closed if issue.closed? + + close_issue(issue: issue) + + presenter(issue).present + end + + private + + def close_issue(issue:) + Issues::CloseService.new(project, current_user).execute(issue) + end + + def presenter(issue) + Gitlab::SlashCommands::Presenters::IssueClose.new(issue) + end + + def not_found + Gitlab::SlashCommands::Presenters::Access.new.not_found + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index b6db103b82b..08cb82274fd 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -40,6 +40,14 @@ module Gitlab ] end + def project_link + "[#{project.full_name}](#{project.web_url})" + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + private attr_reader :resource diff --git a/lib/gitlab/slash_commands/presenters/issue_close.rb b/lib/gitlab/slash_commands/presenters/issue_close.rb new file mode 100644 index 00000000000..b3f24f4296a --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/issue_close.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + class IssueClose < Presenters::Base + include Presenters::IssueBase + + def present + if @resource.confidential? + ephemeral_response(close_issue) + else + in_channel_response(close_issue) + end + end + + def already_closed + ephemeral_response(text: "Issue #{@resource.to_reference} is already closed.") + end + + private + + def close_issue + { + attachments: [ + { + title: "#{@resource.title} ยท #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "Closed issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :fields + ] + } + ] + } + end + + def pretext + "I closed an issue on #{author_profile_link}'s behalf: *#{@resource.to_reference}* in #{project_link}" + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index ac78745ae70..1424a4ac381 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -36,15 +36,7 @@ module Gitlab end def pretext - "I created an issue on #{author_profile_link}'s behalf: **#{@resource.to_reference}** in #{project_link}" - end - - def project_link - "[#{project.full_name}](#{project.web_url})" - end - - def author_profile_link - "[#{author.to_reference}](#{url_for(author)})" + "I created an issue on #{author_profile_link}'s behalf: *#{@resource.to_reference}* in #{project_link}" end end end diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb deleted file mode 100644 index 9f12513e09e..00000000000 --- a/lib/gitlab/snowplow_tracker.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'snowplow-tracker' - -module Gitlab - module SnowplowTracker - NAMESPACE = 'cf' - - class << self - def track_event(category, action, label: nil, property: nil, value: nil, context: nil) - tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i) - end - - private - - def tracker - return unless enabled? - - @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id) - end - - def subject - ::SnowplowTracker::Subject.new - end - - def emitter - ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname) - end - - def enabled? - Gitlab::CurrentSettings.snowplow_enabled? - end - end - end -end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb new file mode 100644 index 00000000000..ef669b03c87 --- /dev/null +++ b/lib/gitlab/tracking.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module Tracking + SNOWPLOW_NAMESPACE = 'gl' + + class << self + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + + def event(category, action, label: nil, property: nil, value: nil, context: nil) + return unless enabled? + + snowplow.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + end + + def snowplow_options(group) + additional_features = Feature.enabled?(:additional_snowplow_tracking, group) + { + namespace: SNOWPLOW_NAMESPACE, + hostname: Gitlab::CurrentSettings.snowplow_collector_hostname, + cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, + app_id: Gitlab::CurrentSettings.snowplow_site_id, + page_tracking_enabled: additional_features, + activity_tracking_enabled: additional_features + }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } + end + + private + + def snowplow + @snowplow ||= SnowplowTracker::Tracker.new( + SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname), + SnowplowTracker::Subject.new, + SNOWPLOW_NAMESPACE, + Gitlab::CurrentSettings.snowplow_site_id + ) + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 1542905d2ce..a93301cb4ce 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -142,7 +142,8 @@ module Gitlab Gitlab::UsageDataCounters::SnippetCounter, Gitlab::UsageDataCounters::SearchCounter, Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter + Gitlab::UsageDataCounters::SourceCodeCounter, + Gitlab::UsageDataCounters::MergeRequestCounter ] end @@ -188,8 +189,8 @@ module Gitlab {} # augmented in EE end - def count(relation, fallback: -1) - relation.count + def count(relation, count_by: nil, fallback: -1) + count_by ? relation.count(count_by) : relation.count rescue ActiveRecord::StatementInvalid fallback end diff --git a/lib/gitlab/usage_data_counters/merge_request_counter.rb b/lib/gitlab/usage_data_counters/merge_request_counter.rb new file mode 100644 index 00000000000..e786e595f77 --- /dev/null +++ b/lib/gitlab/usage_data_counters/merge_request_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class MergeRequestCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'merge_request' + end + end +end diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb index e93a0bcfa27..672450ec82b 100644 --- a/lib/gitlab/usage_data_counters/note_counter.rb +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -4,7 +4,7 @@ module Gitlab::UsageDataCounters class NoteCounter < BaseCounter KNOWN_EVENTS = %w[create].freeze PREFIX = 'note' - COUNTABLE_TYPES = %w[Snippet].freeze + COUNTABLE_TYPES = %w[Snippet Commit MergeRequest].freeze class << self def redis_key(event, noteable_type) @@ -24,9 +24,9 @@ module Gitlab::UsageDataCounters end def totals - { - snippet_comment: read(:create, 'Snippet') - } + COUNTABLE_TYPES.map do |countable_type| + [:"#{countable_type.underscore}_comment", read(:create, countable_type)] + end.to_h end private diff --git a/lib/gitlab/visibility_level_checker.rb b/lib/gitlab/visibility_level_checker.rb new file mode 100644 index 00000000000..f15f1486a4e --- /dev/null +++ b/lib/gitlab/visibility_level_checker.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Gitlab::VisibilityLevelChecker verifies that: +# - Current @project.visibility_level is not restricted +# - Override visibility param is not restricted +# - @see https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file +# +# @param current_user [User] Current user object to verify visibility level against +# @param project [Project] Current project that is being created/imported +# @param project_params [Hash] Supplementary project params (e.g. import +# params containing visibility override) +# +# @example +# user = User.find(2) +# project = Project.last +# project_params = {:import_data=>{:data=>{:override_params=>{"visibility"=>"public"}}}} +# level_checker = Gitlab::VisibilityLevelChecker.new(user, project, project_params: project_params) +# +# project_visibility = level_checker.level_restricted? +# => #<Gitlab::VisibilityEvaluationResult:0x00007fbe16ee33c0 @restricted=true, @visibility_level=20> +# +# if project_visibility.restricted? +# deny_visibility_level(project, project_visibility.visibility_level) +# end +# +# @return [VisibilityEvaluationResult] Visibility evaluation result. Responds to: +# #restricted - boolean indicating if level is restricted +# #visibility_level - integer of restricted visibility level +# +module Gitlab + class VisibilityLevelChecker + def initialize(current_user, project, project_params: {}) + @current_user = current_user + @project = project + @project_params = project_params + end + + def level_restricted? + return VisibilityEvaluationResult.new(true, override_visibility_level_value) if override_visibility_restricted? + return VisibilityEvaluationResult.new(true, project.visibility_level) if project_visibility_restricted? + + VisibilityEvaluationResult.new(false, nil) + end + + private + + attr_reader :current_user, :project, :project_params + + def override_visibility_restricted? + return unless import_data + return unless override_visibility_level + return if Gitlab::VisibilityLevel.allowed_for?(current_user, override_visibility_level_value) + + true + end + + def project_visibility_restricted? + return if Gitlab::VisibilityLevel.allowed_for?(current_user, project.visibility_level) + + true + end + + def import_data + @import_data ||= project_params[:import_data] + end + + def override_visibility_level + @override_visibility_level ||= import_data.deep_symbolize_keys.dig(:data, :override_params, :visibility) + end + + def override_visibility_level_value + @override_visibility_level_value ||= Gitlab::VisibilityLevel.level_value(override_visibility_level) + end + end + + class VisibilityEvaluationResult + attr_reader :visibility_level + + def initialize(restricted, visibility_level) + @restricted = restricted + @visibility_level = visibility_level + end + + def restricted? + @restricted + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3b77fe838ae..29087d26007 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -34,7 +34,8 @@ module Gitlab GitConfigOptions: [], GitalyServer: { address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) + token: Gitlab::GitalyClient.token(project.repository_storage), + features: Feature::Gitaly.server_feature_flags } } @@ -250,7 +251,8 @@ module Gitlab def gitaly_server_hash(repository) { address: Gitlab::GitalyClient.address(repository.project.repository_storage), - token: Gitlab::GitalyClient.token(repository.project.repository_storage) + token: Gitlab::GitalyClient.token(repository.project.repository_storage), + features: Feature::Gitaly.server_feature_flags } end diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb deleted file mode 100644 index 99be51bc8c6..00000000000 --- a/lib/gt_one_coercion.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class GtOneCoercion < Virtus::Attribute - def coerce(value) - [1, value.to_i].max - end -end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 2d78818630d..a35783c1971 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -3,6 +3,24 @@ module Peek module Views class ActiveRecord < DetailedView + DEFAULT_THRESHOLDS = { + calls: 100, + duration: 3, + individual_call: 1 + }.freeze + + THRESHOLDS = { + production: { + calls: 100, + duration: 15, + individual_call: 5 + } + }.freeze + + def self.thresholds + @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) + end + private def setup_subscribers diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb index f4ca1cb5075..4f3eddaf11b 100644 --- a/lib/peek/views/detailed_view.rb +++ b/lib/peek/views/detailed_view.rb @@ -3,11 +3,16 @@ module Peek module Views class DetailedView < View + def self.thresholds + {} + end + def results { - duration: formatted_duration, + duration: format_duration(duration), calls: calls, - details: details + details: details, + warnings: warnings } end @@ -18,30 +23,48 @@ module Peek private def duration - detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord + detail_store.map { |entry| entry[:duration] }.sum * 1000 # rubocop:disable CodeReuse/ActiveRecord end def calls detail_store.count end + def details + call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def warnings + [ + warning_for(calls, self.class.thresholds[:calls], label: "#{key} calls"), + warning_for(duration, self.class.thresholds[:duration], label: "#{key} duration") + ].flatten.compact + end + def call_details detail_store end def format_call_details(call) - call.merge(duration: (call[:duration] * 1000).round(3)) - end + duration = (call[:duration] * 1000).round(3) - def details - call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) + call.merge(duration: duration, + warnings: warning_for(duration, self.class.thresholds[:individual_call])) end - def formatted_duration - ms = duration * 1000 + def warning_for(actual, threshold, label: nil) + if threshold && actual > threshold + prefix = "#{label}: " if label + + ["#{prefix}#{actual} over #{threshold}"] + else + [] + end + end + def format_duration(ms) if ms >= 1000 "%.2fms" % ms else diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index 6ad6ddfd89d..f669feae254 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -3,6 +3,24 @@ module Peek module Views class Gitaly < DetailedView + DEFAULT_THRESHOLDS = { + calls: 30, + duration: 1, + individual_call: 0.5 + }.freeze + + THRESHOLDS = { + production: { + calls: 30, + duration: 1, + individual_call: 0.5 + } + }.freeze + + def self.thresholds + @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) + end + private def duration diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb index 18b3f422852..3ed54a010f8 100644 --- a/lib/peek/views/rugged.rb +++ b/lib/peek/views/rugged.rb @@ -12,7 +12,7 @@ module Peek private def duration - ::Gitlab::RuggedInstrumentation.query_time + ::Gitlab::RuggedInstrumentation.query_time_ms end def calls diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb new file mode 100644 index 00000000000..6418b4de166 --- /dev/null +++ b/lib/prometheus/cleanup_multiproc_dir_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Prometheus + class CleanupMultiprocDirService + include Gitlab::Utils::StrongMemoize + + def execute + FileUtils.rm_rf(old_metrics) if old_metrics + end + + private + + def old_metrics + strong_memoize(:old_metrics) do + Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir + end + end + + def multiprocess_files_dir + ::Prometheus::Client.configuration.multiprocess_files_dir + end + end +end diff --git a/lib/system_check/app/authorized_keys_permission_check.rb b/lib/system_check/app/authorized_keys_permission_check.rb new file mode 100644 index 00000000000..1246a6875a3 --- /dev/null +++ b/lib/system_check/app/authorized_keys_permission_check.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module SystemCheck + module App + class AuthorizedKeysPermissionCheck < SystemCheck::BaseCheck + set_name 'Is authorized keys file accessible?' + set_skip_reason 'skipped (authorized keys not enabled)' + + def skip? + !authorized_keys_enabled? + end + + def check? + authorized_keys.accessible? + end + + def repair! + authorized_keys.create + end + + def show_error + try_fixing_it([ + "sudo chmod 700 #{File.dirname(authorized_keys.file)}", + "touch #{authorized_keys.file}", + "sudo chmod 600 #{authorized_keys.file}" + ]) + fix_and_rerun + end + + private + + def authorized_keys_enabled? + Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + end + + def authorized_keys + @authorized_keys ||= Gitlab::AuthorizedKeys.new + end + end + end +end diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb index cc32feb8604..e98cee510ff 100644 --- a/lib/system_check/rake_task/app_task.rb +++ b/lib/system_check/rake_task/app_task.rb @@ -30,7 +30,8 @@ module SystemCheck SystemCheck::App::RubyVersionCheck, SystemCheck::App::GitVersionCheck, SystemCheck::App::GitUserDefaultSSHConfigCheck, - SystemCheck::App::ActiveUsersCheck + SystemCheck::App::ActiveUsersCheck, + SystemCheck::App::AuthorizedKeysPermissionCheck ] end end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index a07ae3a418a..7a42e4e92a0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -10,15 +10,9 @@ namespace :gitlab do rake:assets:precompile webpack:compile gitlab:assets:fix_urls - gitlab:assets:compile_vrt ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) end - desc 'GitLab | Assets | Compile visual review toolbar' - task :compile_vrt do - system 'yarn', 'webpack-vrt' - end - desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/lib/tasks/gitlab/uploads/sanitize.rake b/lib/tasks/gitlab/uploads/sanitize.rake index 12cf5302555..4f23a0a5d82 100644 --- a/lib/tasks/gitlab/uploads/sanitize.rake +++ b/lib/tasks/gitlab/uploads/sanitize.rake @@ -2,7 +2,7 @@ namespace :gitlab do namespace :uploads do namespace :sanitize do desc 'GitLab | Uploads | Remove EXIF from images.' - task :remove_exif, [:start_id, :stop_id, :dry_run, :sleep_time] => :environment do |task, args| + task :remove_exif, [:start_id, :stop_id, :dry_run, :sleep_time, :uploader, :since] => :environment do |task, args| args.with_defaults(dry_run: 'true') args.with_defaults(sleep_time: 0.3) @@ -11,7 +11,9 @@ namespace :gitlab do sanitizer = Gitlab::Sanitizers::Exif.new(logger: logger) sanitizer.batch_clean(start_id: args.start_id, stop_id: args.stop_id, dry_run: args.dry_run != 'false', - sleep_time: args.sleep_time.to_f) + sleep_time: args.sleep_time.to_f, + uploader: args.uploader, + since: args.since) end end end |