diff options
Diffstat (limited to 'lib')
175 files changed, 2260 insertions, 665 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index bf8ddba6f0d..20f8c637274 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,7 +6,7 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") - NO_SLASH_URL_PART_REGEX = %r{[^/]+} + NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze @@ -98,7 +98,6 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages - mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistry @@ -134,6 +133,7 @@ module API mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectClusters + mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 5c98b0ad56c..65d7f68bbf9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -162,8 +162,8 @@ module API result = DeleteBranchService.new(user_project, current_user) .execute(params[:branch]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result.error? + render_api_error!(result.message, result.http_status) end end end diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb deleted file mode 100644 index da756daadcc..00000000000 --- a/lib/api/circuit_breakers.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module API - class CircuitBreakers < Grape::API - before { authenticated_as_admin! } - - resource :circuit_breakers do - params do - requires :type, - type: String, - desc: "The type of circuitbreaker", - values: ['repository_storage'] - end - resource ':type' do - namespace '', requirements: { type: 'repository_storage' } do - desc 'Get all git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get do - present [] - end - - desc 'Get all failing git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get 'failing' do - present [] - end - - desc 'Reset all storage failures and open circuitbreaker' do - detail 'This feature was introduced in GitLab 9.5' - end - delete do - end - end - end - end - end -end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 8afe6dda414..5928ee1657b 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -134,9 +134,13 @@ module API post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) + first_note = notes.first break not_found!("Discussion") if notes.empty? - break bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? + + unless first_note.part_of_discussion? || first_note.to_discussion.can_convert_to_discussion? + break bad_request!("Discussion can not be replied to.") + end opts = { note: params[:body], diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ee8480122c4..96a1ccefbe5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -542,10 +542,15 @@ module API class IssueBasic < ProjectEntity expose :closed_at expose :closed_by, using: Entities::UserBasic - expose :labels do |issue| - # Avoids an N+1 query since labels are preloaded - issue.labels.map(&:title).sort + + expose :labels do |issue, options| + if options[:with_labels_details] + ::API::Entities::LabelBasic.represent(issue.labels.sort_by(&:title)) + else + issue.labels.map(&:title).sort + end end + expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic @@ -573,6 +578,14 @@ module API class Issue < IssueBasic include ::API::Helpers::RelatedResourcesHelpers + expose(:has_tasks) do |issue, _| + !issue.task_list_items.empty? + end + + expose :task_status, if: -> (issue, _) do + !issue.task_list_items.empty? + end + expose :_links do expose :self do |issue| expose_url(api_v4_project_issue_path(id: issue.project_id, issue_iid: issue.iid)) @@ -878,7 +891,7 @@ module API expose :push_event_payload, as: :push_data, using: PushEventPayload, - if: -> (event, _) { event.push? } + if: -> (event, _) { event.push_action? } expose :author_username do |event, options| event.author&.username @@ -1156,22 +1169,33 @@ module API end end - class Release < TagRelease + class Release < Grape::Entity expose :name + expose :tag, as: :tag_name, if: lambda { |_, _| can_download_code? } + expose :description expose :description_html do |entity| MarkupHelper.markdown_field(entity, :description) end expose :created_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } - expose :commit, using: Entities::Commit + expose :commit, using: Entities::Commit, if: lambda { |_, _| can_download_code? } expose :assets do - expose :assets_count, as: :count - expose :sources, using: Entities::Releases::Source + expose :assets_count, as: :count do |release, _| + assets_to_exclude = can_download_code? ? [] : [:sources] + release.assets_count(except: assets_to_exclude) + end + expose :sources, using: Entities::Releases::Source, if: lambda { |_, _| can_download_code? } expose :links, using: Entities::Releases::Link do |release, options| release.links.sorted end end + + private + + def can_download_code? + Ability.allowed?(options[:current_user], :download_code, object.project) + end end class Tag < Grape::Entity @@ -1242,7 +1266,7 @@ module API end class JobBasic < Grape::Entity - expose :id, :status, :stage, :name, :ref, :tag, :coverage + expose :id, :status, :stage, :name, :ref, :tag, :coverage, :allow_failure expose :created_at, :started_at, :finished_at expose :duration expose :user, with: User @@ -1277,8 +1301,9 @@ module API end class Variable < Grape::Entity - expose :key, :value + expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } + expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } end class Pipeline < PipelineBasic diff --git a/lib/api/events.rb b/lib/api/events.rb index b98aa9f31e1..e4c017fab42 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -4,34 +4,11 @@ module API class Events < Grape::API include PaginationParams include APIGuard + helpers ::API::Helpers::EventsHelpers - helpers do - params :event_filter_params do - optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' - optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' - optional :before, type: Date, desc: 'Include only events created before this date' - optional :after, type: Date, desc: 'Include only events created after this date' - end - - params :sort_params do - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return events sorted in ascending and descending order' - end - - def present_events(events) - events = paginate(events) - - present events, with: Entities::Event - end - - def find_events(source) - EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute - end - end + allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :events do - allow_access_with_scope :read_user, if: -> (request) { request.get? } - desc "List currently authenticated user's events" do detail 'This feature was introduced in GitLab 9.3.' success Entities::Event @@ -55,8 +32,6 @@ module API requires :id, type: String, desc: 'The ID or Username of the user' end resource :users do - allow_access_with_scope :read_user, if: -> (request) { request.get? } - desc 'Get the contribution events of a specified user' do detail 'This feature was introduced in GitLab 8.13.' success Entities::Event @@ -76,25 +51,5 @@ module API present_events(events) end end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "List a Project's visible events" do - success Entities::Event - end - params do - use :pagination - use :event_filter_params - use :sort_params - end - - get ":id/events" do - events = find_events(user_project) - - present_events(events) - end - end end end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 3f048e0dc56..47fcbabb4d4 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -47,6 +47,7 @@ module API requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -67,6 +68,7 @@ module API optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ad16f26f5cc..ec1020c7c78 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,35 +7,9 @@ module API before { authenticate_non_get! } - helpers do - params :optional_params_ce do - optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, - values: Gitlab::VisibilityLevel.string_values, - default: Gitlab::VisibilityLevel.string_level( - Gitlab::CurrentSettings.current_application_settings.default_group_visibility), - desc: 'The visibility of the group' - optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' - end - - if Gitlab.ee? - params :optional_params_ee do - optional :membership_lock, type: Boolean, desc: 'Prevent adding new members to project membership within this group' - optional :ldap_cn, type: String, desc: 'LDAP Common Name' - optional :ldap_access, type: Integer, desc: 'A valid access level' - optional :shared_runners_minutes_limit, type: Integer, desc: '(admin-only) Pipeline minutes quota for this group' - optional :extra_shared_runners_minutes_limit, type: Integer, desc: '(admin-only) Extra pipeline minutes quota for this group' - all_or_none_of :ldap_cn, :ldap_access - end - end - - params :optional_params do - use :optional_params_ce - use :optional_params_ee if Gitlab.ee? - end + helpers Helpers::GroupsHelpers + helpers do params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end @@ -176,10 +150,7 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - - if Gitlab.ee? - optional :file_template_project_id, type: Integer, desc: 'The ID of a project to use for custom templates in this group' - end + use :optional_update_params_ee end put ':id' do group = find_group!(params[:id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8a21d44b4bf..7e4539d0419 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -67,10 +67,6 @@ module API initial_current_user != current_user end - def user_namespace - @user_namespace ||= find_namespace!(params[:id]) - end - def user_group @group ||= find_group!(params[:id]) end diff --git a/lib/api/helpers/events_helpers.rb b/lib/api/helpers/events_helpers.rb new file mode 100644 index 00000000000..bf3b76bb92d --- /dev/null +++ b/lib/api/helpers/events_helpers.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module API + module Helpers + module EventsHelpers + extend Grape::API::Helpers + + params :event_filter_params do + optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' + optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' + optional :before, type: Date, desc: 'Include only events created before this date' + optional :after, type: Date, desc: 'Include only events created after this date' + end + + params :sort_params do + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return events sorted in ascending and descending order' + end + + def present_events(events) + events = paginate(events) + + present events, with: Entities::Event + end + + def find_events(source) + EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute + end + end + end +end diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb new file mode 100644 index 00000000000..2c33d79f6c8 --- /dev/null +++ b/lib/api/helpers/groups_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module API + module Helpers + module GroupsHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ce do + optional :description, type: String, desc: 'The description of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' + end + + params :optional_params_ee do + end + + params :optional_update_params_ee do + end + + params :optional_params do + use :optional_params_ce + use :optional_params_ee + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 71c30ec99a5..c318f5b9127 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -46,6 +46,8 @@ module API def process_mr_push_options(push_options, project, user, changes) output = {} + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/61359') + service = ::MergeRequests::PushOptionsHandlerService.new( project, user, diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index f6762910b0c..fc66cec5341 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -18,6 +18,39 @@ module API :title ] end + + def issue_finder(args = {}) + args = declared_params.merge(args) + + args.delete(:id) + args[:milestone_title] ||= args.delete(:milestone) + args[:label_name] ||= args.delete(:labels) + args[:scope] = args[:scope].underscore if args[:scope] + + IssuesFinder.new(current_user, args) + end + + def find_issues(args = {}) + finder = issue_finder(args) + issues = finder.execute.with_api_entity_associations + + issues.reorder(order_options_with_tie_breaker) # rubocop: disable CodeReuse/ActiveRecord + end + + def issues_statistics(args = {}) + finder = issue_finder(args) + counter = Gitlab::IssuablesCountForState.new(finder) + + { + statistics: { + counts: { + all: counter[:all], + closed: counter[:closed], + opened: counter[:opened] + } + } + } + end end end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 73d58ee7f37..1395ffadab9 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -19,28 +19,13 @@ module API .non_request end - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_project(project) - shared_group_ids = project.project_group_links.pluck(:group_id) - project_group_ids = project.group&.self_and_ancestors&.pluck(:id) - source_ids = [project.id, project_group_ids, shared_group_ids] - .flatten - .compact - Member.includes(:user) - .joins(user: :project_authorizations) - .where(project_authorizations: { project_id: project.id }) - .where(source_id: source_ids) + MembersFinder.new(project, current_user).execute(include_invited_groups_members: true) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_group(group) - source_ids = group.self_and_ancestors.pluck(:id) - Member.includes(:user) - .where(source_id: source_ids) - .where(source_type: 'Namespace') + GroupMembersFinder.new(group).execute end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index 793ae11b41d..9cdde25fe4e 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -13,6 +13,10 @@ module API available?(:merge_requests, project, options[:current_user]) end + def expose_path(path) + Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, path) + end + def expose_url(path) url_options = Gitlab::Application.routes.default_url_options protocol, host, port, script_name = url_options.values_at(:protocol, :host, :port, :script_name) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 00f0bbab231..224aaaaf006 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -264,10 +264,8 @@ module API PostReceive.perform_async(params[:gl_repository], params[:identifier], params[:changes], push_options.as_json) - if Feature.enabled?(:mr_push_options, default_enabled: true) - mr_options = push_options.get(:merge_request) - output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? - end + mr_options = push_options.get(:merge_request) + output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d0a93b77951..0b4da01f3c8 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,27 +3,12 @@ module API class Issues < Grape::API include PaginationParams + helpers Helpers::IssuesHelpers + helpers ::Gitlab::IssuableMetadata before { authenticate_non_get! } - helpers ::Gitlab::IssuableMetadata - helpers do - # rubocop: disable CodeReuse/ActiveRecord - def find_issues(args = {}) - args = declared_params.merge(args) - - args.delete(:id) - args[:milestone_title] = args.delete(:milestone) - args[:label_name] = args.delete(:labels) - args[:scope] = args[:scope].underscore if args[:scope] - - issues = IssuesFinder.new(current_user, args).execute - .with_api_entity_associations - issues.reorder(order_options_with_tie_breaker) - end - # rubocop: enable CodeReuse/ActiveRecord - if Gitlab.ee? params :issues_params_ee do optional :weight, types: [Integer, String], integer_none_any: true, desc: 'The weight of the issue' @@ -34,13 +19,9 @@ module API end end - params :issues_params do + params :issues_stats_params do optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return issues ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' optional :search, type: String, desc: 'Search issues for text present in the title, description, or any combination of these' @@ -49,18 +30,39 @@ module API optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' + optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' + optional :author_username, type: String, desc: 'Return issues which are authored by the user with the given username' + mutually_exclusive :author_id, :author_username + optional :assignee_id, types: [Integer, String], integer_none_any: true, desc: 'Return issues which are assigned to the user with the given ID' + optional :assignee_username, type: Array[String], check_assignees_count: true, + coerce_with: Validations::CheckAssigneesCount.coerce, + desc: 'Return issues which are assigned to the user with the given username' + mutually_exclusive :assignee_id, :assignee_username + optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' - use :pagination use :issues_params_ee if Gitlab.ee? end + params :issues_params do + optional :with_labels_details, type: Boolean, desc: 'Return more label data than just lable title', default: false + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + + use :issues_stats_params + use :pagination + end + params :issue_params do optional :description, type: String, desc: 'The description of an issue' optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' @@ -75,13 +77,23 @@ module API end end + desc "Get currently authenticated user's issues statistics" + params do + use :issues_stats_params + optional :scope, type: String, values: %w[created_by_me assigned_to_me all], default: 'created_by_me', + desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' + end + get '/issues_statistics' do + authenticate! unless params[:scope] == 'all' + + present issues_statistics, with: Grape::Presenters::Presenter + end + resource :issues do desc "Get currently authenticated user's issues" do - success Entities::IssueBasic + success Entities::Issue end params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' use :issues_params optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me', desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' @@ -91,7 +103,8 @@ module API issues = paginate(find_issues) options = { - with: Entities::IssueBasic, + with: Entities::Issue, + with_labels_details: declared_params[:with_labels_details], current_user: current_user, issuable_metadata: issuable_meta_data(issues, 'Issue') } @@ -105,11 +118,9 @@ module API end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group issues' do - success Entities::IssueBasic + success Entities::Issue end params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' use :issues_params end get ":id/issues" do @@ -118,13 +129,24 @@ module API issues = paginate(find_issues(group_id: group.id, include_subgroups: true)) options = { - with: Entities::IssueBasic, + with: Entities::Issue, + with_labels_details: declared_params[:with_labels_details], current_user: current_user, issuable_metadata: issuable_meta_data(issues, 'Issue') } present issues, options end + + desc 'Get statistics for the list of group issues' + params do + use :issues_stats_params + end + get ":id/issues_statistics" do + group = find_group!(params[:id]) + + present issues_statistics(group_id: group.id, include_subgroups: true), with: Grape::Presenters::Presenter + end end params do @@ -134,11 +156,9 @@ module API include TimeTrackingEndpoints desc 'Get a list of project issues' do - success Entities::IssueBasic + success Entities::Issue end params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' use :issues_params end get ":id/issues" do @@ -147,7 +167,8 @@ module API issues = paginate(find_issues(project_id: project.id)) options = { - with: Entities::IssueBasic, + with: Entities::Issue, + with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, issuable_metadata: issuable_meta_data(issues, 'Issue') @@ -156,6 +177,16 @@ module API present issues, options end + desc 'Get statistics for the list of project issues' + params do + use :issues_stats_params + end + get ":id/issues_statistics" do + project = find_project!(params[:id]) + + present issues_statistics(project_id: project.id), with: Grape::Presenters::Presenter + end + desc 'Get a single project issue' do success Entities::Issue end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 3cc09f6ac3f..77ecb3e7cde 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -44,6 +44,8 @@ module API requires :id, type: String, desc: "Namespace's ID or path" end get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + user_namespace = find_namespace!(params[:id]) + present user_namespace, with: Entities::Namespace, current_user: current_user end end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index c86b50d3736..1d1ef1afc6b 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -118,6 +118,7 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :variable_type, type: String, values: Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' end post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -138,6 +139,7 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' requires :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :variable_type, type: String, values: Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' end put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :update_pipeline_schedule, pipeline_schedule diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index b62ec887183..dcc8d94fb79 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -54,6 +54,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :domain, type: String, desc: 'Cluster base domain' + optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' requires :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb new file mode 100644 index 00000000000..734311e1142 --- /dev/null +++ b/lib/api/project_events.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + class ProjectEvents < Grape::API + include PaginationParams + include APIGuard + helpers ::API::Helpers::EventsHelpers + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "List a Project's visible events" do + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + + get ":id/events" do + events = find_events(user_project) + + present_events(events) + end + end + end +end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index cb85028f22c..6b17f4317db 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -23,7 +23,7 @@ module API get ':id/releases' do releases = ::ReleasesFinder.new(user_project, current_user).execute - present paginate(releases), with: Entities::Release + present paginate(releases), with: Entities::Release, current_user: current_user end desc 'Get a single project release' do @@ -34,9 +34,9 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag end get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do - authorize_read_release! + authorize_download_code! - present release, with: Entities::Release + present release, with: Entities::Release, current_user: current_user end desc 'Create a new release' do @@ -63,7 +63,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -86,7 +86,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -107,7 +107,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -135,6 +135,10 @@ module API authorize! :destroy_release, release end + def authorize_download_code! + authorize! :download_code, release + end + def release @release ||= user_project.releases.find_by_tag(params[:tag]) end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index ea36c24eca2..fdf4904e9f5 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -98,6 +98,7 @@ module API optional :certificate, type: String, desc: %q(Session's certificate) optional :authorization, type: String, desc: %q(Session's authorization) end + optional :job_age, type: Integer, desc: %q(Job should be older than passed age in seconds to be ran on runner) end post '/request' do authenticate_runner! diff --git a/lib/api/search.rb b/lib/api/search.rb index 60095300ea1..1cab1a97186 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -112,12 +112,13 @@ module API type: String, desc: 'The scope of the search', values: Helpers::SearchHelpers.project_search_scopes + optional :ref, type: String, desc: 'The name of a repository branch or tag. If not given, the default branch is used' use :pagination end get ':id/(-/)search' do check_users_search_allowed! - present search(project_id: user_project.id), with: entity + present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b064747e5fc..8046acfa397 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -150,6 +150,12 @@ module API given elasticsearch_indexing: ->(val) { val } do optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search' requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")' + optional :elasticsearch_limit_indexing, type: Boolean, desc: 'Limit Elasticsearch to index certain namespaces and projects' + end + + given elasticsearch_limit_indexing: ->(val) { val } do + optional :elasticsearch_namespace_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The namespace ids to index with Elasticsearch.' + optional :elasticsearch_project_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The project ids to index with Elasticsearch.' end optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons' diff --git a/lib/api/validations/check_assignees_count.rb b/lib/api/validations/check_assignees_count.rb new file mode 100644 index 00000000000..836ec936b31 --- /dev/null +++ b/lib/api/validations/check_assignees_count.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module API + module Validations + class CheckAssigneesCount < Grape::Validations::Base + def self.coerce + lambda do |value| + case value + when String, Array + Array.wrap(value) + else + [] + end + end + end + + def validate_param!(attr_name, params) + return if param_allowed?(attr_name, params) + + raise Grape::Exceptions::Validation, + params: [@scope.full_name(attr_name)], + message: "allows one value, but found #{params[attr_name].size}: #{params[attr_name].join(", ")}" + end + + private + + def param_allowed?(attr_name, params) + params[attr_name].size <= 1 + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 3489ba827e4..6c61392ed63 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -54,7 +54,9 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' + optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' if Gitlab.ee? optional :environment_scope, type: String, desc: 'The environment_scope of the variable' @@ -79,7 +81,9 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' + optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' if Gitlab.ee? optional :environment_scope, type: String, desc: 'The environment_scope of the variable' diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 994074ddc67..5724adb2c40 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -33,7 +33,8 @@ module API authorize! :read_wiki, user_project entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic - present user_project.wiki.pages, with: entity + + present user_project.wiki.list_pages(load_content: params[:with_content]), with: entity end desc 'Get a wiki page' do diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index 6d01d51955c..cce79e73d2d 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -2,13 +2,13 @@ module Banzai module ColorParser - ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 - PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% - ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ - BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 - DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] - RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad - HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/.freeze # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/.freeze # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/.freeze + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/.freeze # 00..255 + DEGS = /-?\d+(?:deg)?/i.freeze # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i.freeze # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/.freeze RGB_FORMAT = %r{ (?:rgba? \( @@ -20,7 +20,7 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze HSL_FORMAT = %r{ (?:hsla? \( @@ -28,11 +28,11 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze - COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix.freeze # Public: Analyzes whether the String is a color code. # diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 44b151d01e7..0224dd8fcd1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -363,6 +363,14 @@ module Banzai group_ref end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 086adf59d2b..56214043d87 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -33,7 +33,7 @@ module Banzai # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # # Rubular: http://rubular.com/r/nrL3r9yUiq - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)}.freeze # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index a27d18facd1..544231adea4 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -20,7 +20,7 @@ module Banzai \s* ^\k<delim> # closing front matter marker \s* - }mx + }mx.freeze def call html.sub(PATTERN) do |_match| diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 77e4c438bd0..4892668fc22 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -82,16 +82,18 @@ module Banzai def object_link_text(object, matches) label_suffix = '' + parent = project || group if project || full_path_ref?(matches) project_path = full_project_path(matches[:namespace], matches[:project]) parent_from_ref = from_ref_cached(project_path) - reference = parent_from_ref.to_human_reference(project || group) + reference = parent_from_ref.to_human_reference(parent) label_suffix = " <i>in #{reference}</i>" if reference.present? end - LabelsHelper.render_colored_label(object, label_suffix: label_suffix, title: tooltip_title(object)) + presenter = object.present(issuable_subject: parent) + LabelsHelper.render_colored_label(presenter, label_suffix: label_suffix, title: tooltip_title(presenter)) end def tooltip_title(label) @@ -102,14 +104,6 @@ module Banzai matches[:namespace] && matches[:project] end - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end - - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end - def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index fce042e8946..08969753d75 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -51,13 +51,13 @@ module Banzai # default implementation. return super(text, pattern) if pattern != Milestone.reference_pattern - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone yield match, milestone.id, $~[:project], $~[:namespace], $~ else - match + escape_html_entities(match) end end end diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 50bf823929c..ee7f10ebdf6 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -33,7 +33,7 @@ module Banzai (?<new_link>.+?) (?<title>\ ".+?")? \) - }x + }x.freeze # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index f2ae17b44fa..ade4d260be1 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -17,7 +17,7 @@ module Banzai # :toc - String containing Table of Contents data as a `ul` element with # `li` child elements. class TableOfContentsFilter < HTML::Pipeline::Filter - PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u + PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze def call return doc if context[:no_header_anchors] @@ -31,6 +31,7 @@ module Banzai if header_content = node.children.first id = node .text + .strip .downcase .gsub(PUNCTUATION_REGEXP, '') # remove punctuation .tr(' ', '-') # replace spaces with dash diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 7db5f5e1f7d..c2da7fec7cc 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -70,8 +70,11 @@ module Banzai # Build the raw <a> tag just with a link as href and content if # it's originally a link pattern. We shouldn't return a plain text href. original_link = - if link_reference == 'true' && href = original_content - %(<a href="#{href}">#{href}</a>) + if link_reference == 'true' + href = node.attr('href') + content = original_content + + %(<a href="#{href}">#{content}</a>) end # The reference should be replaced by the original link's content, diff --git a/lib/bitbucket_server/representation/repo.rb b/lib/bitbucket_server/representation/repo.rb index 6c494b79166..dab7f8f22a1 100644 --- a/lib/bitbucket_server/representation/repo.rb +++ b/lib/bitbucket_server/representation/repo.rb @@ -20,7 +20,7 @@ module BitbucketServer end def browse_url - # The JSON reponse contains an array of 1 element. Not sure if there + # The JSON response contains an array of 1 element. Not sure if there # are cases where multiple links would be provided. raw.dig('links', 'self').first.fetch('href') end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 239780d8626..9b7d1548056 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module DeclarativePolicy diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 1204e53ee2e..ccaf06c5d6a 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -36,10 +36,11 @@ module Gitlab end COM_URL = 'https://gitlab.com'.freeze - APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} - SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} + APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze + SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze + HTTP_PROXY_ENV_VARS = %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY).freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com @@ -59,7 +60,15 @@ module Gitlab end def self.ee? - Object.const_defined?(:License) + if ENV['IS_GITLAB_EE'].present? + Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) + else + Object.const_defined?(:License) + end + end + + def self.http_proxy_env? + HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] } end def self.process_name diff --git a/lib/gitlab/action_view_output/context.rb b/lib/gitlab/action_view_output/context.rb new file mode 100644 index 00000000000..9fbc9811636 --- /dev/null +++ b/lib/gitlab/action_view_output/context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# This file was simplified from https://raw.githubusercontent.com/rails/rails/195f39804a7a4a0034f25e8704220e03d95a752a/actionview/lib/action_view/context.rb. +# +# It is only needed by modules that need to call ActionView helper +# methods (e.g. those in +# https://github.com/rails/rails/tree/c4d3e202e10ae627b3b9c34498afb45450652421/actionview/lib/action_view/helpers) +# to generate tags outside of a Rails controller (e.g. API, Sidekiq, +# etc.). +# +# In Rails 5, ActionView::Context automatically includes CompiledTemplates. +# This means that any module that includes ActionView::Context is now a descendant +# of CompiledTemplates. +# +# When a partial is rendered for the first time, it runs +# Module#module_eval, which will evaluate a string source that defines a +# new method. For example: +# +# def _app_views_profiles_show_html_haml___1285955918103175884_70307801785400(local_assigns, output_buffer) +# "hello world" +# end +# +# When a new method is defined, the Ruby interpreter clears the method +# cache for all descendants, and all methods for those modules will have +# to be redefined. This can lead to a significant performance penalty. +# +# Rails 6 fixes this behavior by moving out the `include +# CompiledTemplates` into ActionView::Base so that including `ActionView::Context` +# doesn't quietly affect other modules in this way. + +if Rails::VERSION::STRING.start_with?('6') + raise 'This module is no longer needed in Rails 6. Use ActionView::Context instead.' +end + +module Gitlab + module ActionViewOutput + module Context + attr_accessor :output_buffer, :view_flow + end + end +end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 36fc8061d92..72a187377d0 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -55,7 +55,7 @@ module Gitlab private def info - auth_hash.info + auth_hash['info'] end def get_info(key) diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 78fa25c5516..4ebf2afb9cb 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/auth_logger.rb b/lib/gitlab/auth_logger.rb new file mode 100644 index 00000000000..6d3edba02b0 --- /dev/null +++ b/lib/gitlab/auth_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class AuthLogger < Gitlab::JsonLogger + def self.file_name_noext + 'auth' + end + end +end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb index 0a12401c35f..bf5d7f5f322 100644 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -95,7 +95,7 @@ module Gitlab local_labels = available_labels(project_id) # get all label links for the given resource (issue/MR) - # which reference a label not included in avaiable_labels + # which reference a label not included in available_labels # (other than its project labels and labels of ancestor groups) cross_labels = LabelLink .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a84f794bfae..1924f2ffee2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -8,8 +8,8 @@ module Gitlab self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze # These regex patterns are tested against a relative path, relative to # the upload directory. diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 81ca2b0a9b7..1ee44a3a5a9 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -16,7 +16,7 @@ module Gitlab RELATIVE_UPLOAD_DIR ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 769d3279f91..c9f0ed66a54 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -135,7 +135,7 @@ module Gitlab def create_labels LABELS.each do |label_params| - label = ::Labels::CreateService.new(label_params).execute(project: project) + label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) if label.valid? @labels[label_params[:title]] = label else diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 1d3ddeeb0f1..ff2694abd5e 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -201,6 +201,7 @@ module Gitlab target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), target_branch_sha: pull_request.target_branch_sha, state: pull_request.state, + state_id: MergeRequest.available_states[pull_request.state], author_id: author_id, assignee_id: nil, created_at: pull_request.created_at, diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index 1dbd564fb6f..4ddc1c718c7 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -48,7 +48,7 @@ module Gitlab if project.empty_repo? protected_branch_push_checks - elsif creation? && protected_branch_creation_enabled? + elsif creation? protected_branch_creation_checks elsif deletion? protected_branch_deletion_checks @@ -124,10 +124,6 @@ module Gitlab Gitlab::Routing.url_helpers.project_project_members_url(project) end - def protected_branch_creation_enabled? - Feature.enabled?(:protected_branch_creation, project, default_enabled: true) - end - def matching_merge_request? Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index cc6a14d2d9a..67a65d61441 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,6 +7,7 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze def validate! + return unless Feature.enabled?(:lfs_check, default_enabled: true) return unless project.lfs_enabled? return if skip_lfs_integrity_check diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 7011dd1aaf2..1c3ce08be76 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -11,8 +11,8 @@ module Gitlab ParserError = Class.new(StandardError) InvalidStreamError = Class.new(StandardError) - VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ - INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/.freeze + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}.freeze attr_reader :stream, :path, :full_version diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index bb2b209e793..dbdc59505ac 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -7,6 +7,7 @@ module Gitlab class KubernetesNamespace < Base def unmet? deployment_cluster.present? && + deployment_cluster.managed? && !deployment_cluster.project_type? && kubernetes_namespace.new_record? end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 03af99ba9a5..c911bfa7ff6 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb index fe7c8738cc0..68482cf08a9 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/activity.rb @@ -7,11 +7,11 @@ module Gitlab module Limit class Activity < Chain::Base def perform! - # to be overriden in EE + # to be overridden in EE end def break? - false # to be overriden in EE + false # to be overridden in EE end end end diff --git a/lib/gitlab/ci/pipeline/chain/limit/size.rb b/lib/gitlab/ci/pipeline/chain/limit/size.rb index b4d51437cd6..cd330c58406 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/size.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/size.rb @@ -7,11 +7,11 @@ module Gitlab module Limit class Size < Chain::Base def perform! - # to be overriden in EE + # to be overridden in EE end def break? - false # to be overriden in EE + false # to be overridden in EE end end end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 7d6e0704d4a..df92e229f12 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -7,7 +7,7 @@ module Gitlab class Skip < Chain::Base include ::Gitlab::Utils::StrongMemoize - SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i.freeze def perform! if skipped? diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb new file mode 100644 index 00000000000..5fcc9406cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class NotEquals < Lexeme::Operator + PATTERN = /!=/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + @left.evaluate(variables) != @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb new file mode 100644 index 00000000000..14544d33e25 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class NotMatches < Lexeme::Operator + PATTERN = /\!~/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + text = @left.evaluate(variables) + regexp = @right.evaluate(variables) + + regexp.scan(text.to_s).none? + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index f26542361a2..e14edfae51d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -15,7 +15,9 @@ module Gitlab Expression::Lexeme::Pattern, Expression::Lexeme::Null, Expression::Lexeme::Equals, - Expression::Lexeme::Matches + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches ].freeze MAX_TOKENS = 100 diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index b03611f756e..ab5ae9caeea 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -8,13 +8,24 @@ module Gitlab StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ + # presence matchers %w[variable], + + # positive matchers %w[variable equals string], %w[variable equals variable], %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable matches pattern] + %w[variable matches pattern], + + # negative matchers + %w[variable notequals string], + %w[variable notequals variable], + %w[variable notequals null], + %w[string notequals variable], + %w[null notequals variable], + %w[variable notmatches pattern] ].freeze def initialize(statement, variables = {}) diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 58f4642510b..e50b0853725 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -6,7 +6,8 @@ module Gitlab module Stage class Factory < Status::Factory def self.extended_statuses - [Status::SuccessWarning] + [[Status::SuccessWarning], + [Status::Stage::PlayManual]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/stage/play_manual.rb b/lib/gitlab/ci/status/stage/play_manual.rb new file mode 100644 index 00000000000..ac3fc0912fa --- /dev/null +++ b/lib/gitlab/ci/status/stage/play_manual.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Stage + class PlayManual < Status::Extended + include Gitlab::Routing + + def action_icon + 'play' + end + + def action_title + 'Play all manual' + end + + def action_path + pipeline = subject.pipeline + + project_stage_play_manual_path(pipeline.project, pipeline, subject.name) + end + + def action_method + :post + end + + def action_button_title + _('Play all manual') + end + + def self.matches?(stage, user) + stage.manual_playable? + end + + def has_action? + true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index eeefadaa019..f6d240b7b6d 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,14 +1,11 @@ -# Official docker image. -image: docker:latest - -services: - - docker:dind - -before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - build-master: + # Official docker image. + image: docker:latest stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - docker build --pull -t "$CI_REGISTRY_IMAGE" . - docker push "$CI_REGISTRY_IMAGE" @@ -16,7 +13,13 @@ build-master: - master build: + # Official docker image. + image: docker:latest stage: build + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index d36576fe39f..1d55c64ec56 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -338,7 +338,6 @@ rollout 100%: image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} fi - replicas="1" service_enabled="true" postgres_enabled="$POSTGRES_ENABLED" @@ -383,7 +382,7 @@ rollout 100%: --set application.database_url="$DATABASE_URL" \ --set application.secretName="$APPLICATION_SECRET_NAME" \ --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ + --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ @@ -424,7 +423,7 @@ rollout 100%: --set application.database_url="$DATABASE_URL" \ --set application.secretName="$APPLICATION_SECRET_NAME" \ --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ + --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ @@ -433,6 +432,7 @@ rollout 100%: --set postgresql.postgresUser="$POSTGRES_USER" \ --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set postgresql.imageTag="$POSTGRES_VERSION" \ --set application.migrateCommand="$DB_MIGRATE" \ $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ @@ -440,7 +440,9 @@ rollout 100%: chart/ fi - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" + if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then + kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" + fi } function scale() { @@ -490,7 +492,7 @@ rollout 100%: fi helm init --client-only - helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} + helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} if [[ ! -d "$auto_chart" ]]; then helm fetch ${auto_chart} --untar fi @@ -506,23 +508,13 @@ rollout 100%: kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } - # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN - function ensure_kube_ingress_base_domain() { - if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then - export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN - fi - } - function check_kube_domain() { - ensure_kube_ingress_base_domain - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then echo "In order to deploy or use Review Apps," - echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" echo "or by defining a variable at group or project level." echo "You can also manually add it in .gitlab-ci.yml" - echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" false else true diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index eef361c19e9..324e39c7747 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ container_scanning: DOCKER_SERVICE: docker DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.6 + CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 2a90cc9a06c..27a498b2daf 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/project/merge_requests/dast.html +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings @@ -12,44 +12,29 @@ stages: dast: stage: dast - image: docker:stable + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" variables: - DOCKER_DRIVER: overlay2 + # URL to scan: + # DAST_WEBSITE: https://example.com/ + # + # Time limit for target availability (scan is attempted even when timeout): + # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 + # + # Set these variables to scan with an authenticated user: + # DAST_AUTH_URL: https://example.com/sign-in + # DAST_USERNAME: john.doe@example.com + # DAST_PASSWORD: john-doe-password + # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form + # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form + # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between + # + # Perform ZAP Full Scan, which includes both passive and active scanning: + # DAST_FULL_SCAN_ENABLED: "true" allow_failure: true - services: - - docker:stable-dind script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | - function dast_run() { - docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" - } - - | - if [ -n "$DAST_AUTH_URL" ] - then - dast_run \ - --auth-url $DAST_AUTH_URL \ - --auth-username $DAST_USERNAME \ - --auth-password $DAST_PASSWORD \ - --auth-username-field $DAST_USERNAME_FIELD \ - --auth-password-field $DAST_PASSWORD_FIELD - else - dast_run - fi + - /analyze -t $DAST_WEBSITE artifacts: reports: dast: gl-dast-report.json diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 7f80a6e9285..8dd9775c583 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -20,16 +20,27 @@ dependency_scanning: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - | docker run \ - --env DS_ANALYZER_IMAGES \ - --env DS_ANALYZER_IMAGE_PREFIX \ - --env DS_ANALYZER_IMAGE_TAG \ - --env DS_DEFAULT_ANALYZERS \ - --env DEP_SCAN_DISABLE_REMOTE_CHECKS \ - --env DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - --env DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - --env DS_RUN_ANALYZER_TIMEOUT \ + $(propagate_env_vars \ + DS_ANALYZER_IMAGES \ + DS_ANALYZER_IMAGE_PREFIX \ + DS_ANALYZER_IMAGE_TAG \ + DS_DEFAULT_ANALYZERS \ + DS_EXCLUDED_PATHS \ + DEP_SCAN_DISABLE_REMOTE_CHECKS \ + DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + DS_RUN_ANALYZER_TIMEOUT \ + ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index b941e89991e..abf16e5b2e7 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -20,18 +20,31 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - | docker run \ - --env SAST_ANALYZER_IMAGES \ - --env SAST_ANALYZER_IMAGE_PREFIX \ - --env SAST_ANALYZER_IMAGE_TAG \ - --env SAST_DEFAULT_ANALYZERS \ - --env SAST_BRAKEMAN_LEVEL \ - --env SAST_GOSEC_LEVEL \ - --env SAST_FLAWFINDER_LEVEL \ - --env SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - --env SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - --env SAST_RUN_ANALYZER_TIMEOUT \ + $(propagate_env_vars \ + SAST_ANALYZER_IMAGES \ + SAST_ANALYZER_IMAGE_PREFIX \ + SAST_ANALYZER_IMAGE_TAG \ + SAST_DEFAULT_ANALYZERS \ + SAST_EXCLUDED_PATHS \ + SAST_BANDIT_EXCLUDED_PATHS \ + SAST_BRAKEMAN_LEVEL \ + SAST_GOSEC_LEVEL \ + SAST_FLAWFINDER_LEVEL \ + SAST_GITLEAKS_ENTROPY_LEVEL \ + SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ + SAST_RUN_ANALYZER_TIMEOUT \ + ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code diff --git a/lib/gitlab/ci/templates/dotNET-Core.yml b/lib/gitlab/ci/templates/dotNET-Core.yml new file mode 100644 index 00000000000..708b75f83e8 --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET-Core.yml @@ -0,0 +1,116 @@ +# This is a simple example illustrating how to build and test .NET Core project +# with GitLab Continuous Integration / Continuous Delivery. + +# ### Specify the Docker image +# +# Instead of installing .NET Core SDK manually, a docker image is used +# with already pre-installed .NET Core SDK. +# +# The 'latest' tag targets the latest available version of .NET Core SDK image. +# If preferred, you can explicitly specify version of .NET Core (e.g. using '2.2-sdk' tag). +# +# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet +# Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag +# and the Docker itself: https://opensource.com/resources/what-docker +image: microsoft/dotnet:latest + +# ### Define variables +# +variables: + # 1) Name of directory where restore and build objects are stored. + OBJECTS_DIRECTORY: 'obj' + # 2) Name of directory used for keeping restored dependencies. + NUGET_PACKAGES_DIRECTORY: '.nuget' + # 3) A relative path to the source code from project repository root. + # NOTE: Please edit this path so it matches the structure of your project! + SOURCE_CODE_PATH: '*/*/' + +# ### Define stage list +# +# In this example there are only two stages. +# Initially, the project will be built and then tested. +stages: + - build + - test + +# ### Define global cache rule +# +# Before building the project, all dependencies (e.g. third-party NuGet packages) +# must be restored. Jobs on GitLab.com's Shared Runners are executed on autoscaled machines. +# +# Each machine is used only once (for security reasons) and after that is removed. +# This means that, before every job, a dependency restore must be performed +# because restored dependencies are removed along with machines. Fortunately, +# GitLab provides cache mechanism with the aim of keeping restored dependencies +# for other jobs. +# +# This example shows how to configure cache to pass over restored +# dependencies for re-use. +# +# With global cache rule, cached dependencies will be downloaded before every job +# and then unpacked to the paths as specified below. +cache: + # Per-stage and per-branch caching. + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + # Specify three paths that should be cached: + # + # 1) Main JSON file holding information about package dependency tree, packages versions, + # frameworks etc. It also holds information where to the dependencies were restored. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/project.assets.json' + # 2) Other NuGet and MSBuild related files. Also needed. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/*.csproj.nuget.*' + # 3) Path to the directory where restored dependencies are kept. + - '$NUGET_PACKAGES_DIRECTORY' + # + # 'pull-push' policy means that latest cache will be downloaded (if it exists) + # before executing the job, and a newer version will be uploaded afterwards. + # Such a setting saves time when there are no changes in referenced third-party + # packages. + # + # For example, if you run a pipeline with changes in your code, + # but with no changes within third-party packages which your project is using, + # then project restore will happen quickly as all required dependencies + # will already be there — unzipped from cache. + + # 'pull-push' policy is the default cache policy, you do not have to specify it explicitly. + policy: pull-push + +# ### Restore project dependencies +# +# NuGet packages by default are restored to '.nuget/packages' directory +# in the user's home directory. That directory is out of scope of GitLab caching. +# +# To get around this, a custom path can be specified using the '--packages <PATH>' option +# for 'dotnet restore' command. In this example, a temporary directory is created +# in the root of project repository, so its content can be cached. +# +# Learn more about GitLab cache: https://docs.gitlab.com/ee/ci/caching/index.html +before_script: + - 'dotnet restore --packages $NUGET_PACKAGES_DIRECTORY' + +build: + stage: build + # ### Build all projects discovered from solution file. + # + # Note: this will fail if you have any projects in your solution that are not + # .NET Core-based projects (e.g. WCF service), which is based on .NET Framework, + # not .NET Core. In this scenario, you will need to build every .NET Core-based + # project by explicitly specifying a relative path to the directory + # where it is located (e.g. 'dotnet build ./src/ConsoleApp'). + # Only one project path can be passed as a parameter to 'dotnet build' command. + script: + - 'dotnet build --no-restore' + +tests: + stage: test + # ### Run the tests + # + # You can either run tests for all test projects that are defined in your solution + # with 'dotnet test' or run tests only for specific project by specifying + # a relative path to the directory where it is located (e.g. 'dotnet test ./test/UnitTests'). + # + # You may want to define separate testing jobs for different types of testing + # (e.g. integration tests, unit tests etc). + script: + - 'dotnet test --no-restore' diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index bf5f2a31f0e..dfae260239e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -209,10 +209,7 @@ module Gitlab end def paths - [ - default_path, - deprecated_path - ].compact + [default_path] end def default_directory @@ -227,15 +224,6 @@ module Gitlab File.join(default_directory, "#{job.id}.log") end - def deprecated_path - File.join( - Settings.gitlab_ci.builds_path, - job.created_at.utc.strftime("%Y_%m"), - job.project.ci_id.to_s, - "#{job.id}.log" - ) if job.project&.ci_id - end - def trace_artifact job.job_artifacts_trace end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 833aa75adb5..aab10aef398 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -27,13 +27,9 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - # If the `variable_masking` feature is enabled we expose the `masked` - # attribute, otherwise it's not exposed. - # def to_runner_variable @variable.reject do |hash_key, hash_value| - (hash_key == :file && hash_value == false) || - (hash_key == :masked && !Feature.enabled?(:variable_masking)) + hash_key == :file && hash_value == false end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index df34d254c65..6796fcce75f 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -36,10 +36,10 @@ module Gitlab class AllowedArrayValuesValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - unkown_values = value - options[:in] - unless unkown_values.empty? + unknown_values = value - options[:in] + unless unknown_values.empty? record.errors.add(attribute, "contains unknown values: " + - unkown_values.join(', ')) + unknown_values.join(', ')) end end end diff --git a/lib/gitlab/content_disposition.rb b/lib/gitlab/content_disposition.rb index 32207514ce5..ff6154a5b26 100644 --- a/lib/gitlab/content_disposition.rb +++ b/lib/gitlab/content_disposition.rb @@ -22,13 +22,13 @@ module Gitlab end # rubocop:disable Style/VariableInterpolation - TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/.freeze def ascii_filename 'filename="' + percent_escape(::I18n.transliterate(filename), TRADITIONAL_ESCAPED_CHAR) + '"' end - RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/.freeze # rubocop:enable Style/VariableInterpolation def utf8_filename diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d347f3c13a4..f0ca397609d 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -99,15 +99,13 @@ module Gitlab end CATEGORY_LABELS = { - docs: "~Documentation", + docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", qa: "~QA" }.freeze - - # rubocop:disable Style/RegexpLiteral CATEGORIES = { - %r{\Adoc/} => :docs, - %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, + %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. + %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\A(ee/)?app/(assets|views)/} => :frontend, %r{\A(ee/)?public/} => :frontend, @@ -123,6 +121,8 @@ module Gitlab \.prettierrc | \.scss-lint.yml | \.stylelintrc | + \.haml-lint.yml | + \.haml-lint_todo.yml | babel\.config\.js | jest\.config\.js | karma\.config\.js | @@ -148,10 +148,9 @@ module Gitlab # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, - %r{\.(md|txt)\z} => :docs, + %r{\.(md|txt)\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\.js\z} => :frontend }.freeze - # rubocop:enable Style/RegexpLiteral end end end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb new file mode 100644 index 00000000000..f11e032ab84 --- /dev/null +++ b/lib/gitlab/data_builder/deployment.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Deployment + extend self + + def build(deployment) + { + object_kind: 'deployment', + status: deployment.status, + deployable_id: deployment.deployable_id, + deployable_url: Gitlab::UrlBuilder.build(deployment.deployable), + environment: deployment.environment.name, + project: deployment.project.hook_attrs, + short_sha: deployment.short_sha, + user: deployment.user.hook_attrs, + user_url: Gitlab::UrlBuilder.build(deployment.user), + commit_url: Gitlab::UrlBuilder.build(deployment.commit), + commit_title: deployment.commit.title + } + end + end + end +end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index fa06fb935f7..e1e813849bf 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -19,7 +19,7 @@ module Gitlab def hook_attrs(pipeline) { id: pipeline.id, - ref: pipeline.ref, + ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, before_sha: pipeline.before_sha, diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index af385d7d4ca..40bda3410e1 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -58,7 +58,10 @@ module Gitlab # } # # rubocop:disable Metrics/ParameterLists - def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: {}) + def build( + project:, user:, ref:, oldrev: nil, newrev: nil, + commits: [], commits_count: nil, message: nil, push_options: {}) + commits = Array(commits) # Total commits count @@ -113,7 +116,12 @@ module Gitlab ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" commits = project.repository.commits(project.default_branch.to_s, limit: 3) - build(project, user, commits.last&.id, commits.first&.id, ref, commits) + build(project: project, + user: user, + oldrev: commits.last&.id, + newrev: commits.first&.id, + ref: ref, + commits: commits) end def sample_data diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7f5eb1188fc..cc61bb7fa02 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -905,6 +905,12 @@ module Gitlab end end + def remove_foreign_key_if_exists(*args) + if foreign_key_exists?(*args) + remove_foreign_key(*args) + end + end + def remove_foreign_key_without_error(*args) remove_foreign_key(*args) rescue ArgumentError diff --git a/lib/gitlab/diff/suggestion.rb b/lib/gitlab/diff/suggestion.rb index 027c7a31bcf..4a3ac2106e2 100644 --- a/lib/gitlab/diff/suggestion.rb +++ b/lib/gitlab/diff/suggestion.rb @@ -33,6 +33,8 @@ module Gitlab end def to_content + return "" if @text.blank? + # The parsed suggestion doesn't have information about the correct # ending characters (we may have a line break, or not), so we take # this information from the last line being changed (last diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 270cfb89488..369c6b87fb4 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -52,6 +52,19 @@ module Gitlab end end + # Clears multiple cache keys at once. + # + # raw_keys - An Array of unique cache keys, without namespaces. + # + # It returns the number of cache keys cleared. Ex.: 42 + def clear_multiple(raw_keys) + return [] if raw_keys.empty? + + keys = raw_keys.map { |id| cache_key_for(id) } + + Redis::Cache.with { |redis| redis.del(keys) } + end + def cache_key_for(raw_key) "#{cache_key_prefix}:#{raw_key}" end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 2770469ca9f..9fc2217ad43 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -16,6 +16,7 @@ module Gitlab avatar: /\Alogo\.(png|jpg|gif)\z/, issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, + metrics_dashboard: %r{\A\.gitlab/dashboards/[^/]+\.yml\z}, xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z}, # Configuration files diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 8eb3c28ab70..d0577d7a4ff 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -40,6 +40,10 @@ module Gitlab @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end + def fetch + object_pool_service.fetch(source_repository) + end + private def object_pool_service diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index b46d4ba0b02..ef9b1bf5224 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -14,7 +14,7 @@ module Gitlab 'GL-HOOK-ERR:' # Messages marked as safe by user ].freeze - SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/ + SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze def initialize(message = '') super(sanitize(message)) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a22e3c4b9dd..455588f3c66 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -118,6 +118,12 @@ module Gitlab gitaly_repository_client.exists? end + def create_repository + wrapped_gitaly_errors do + gitaly_repository_client.create_repository + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -494,7 +500,7 @@ module Gitlab end # Return total diverging commits count - def diverging_commit_count(from, to, max_count:) + def diverging_commit_count(from, to, max_count: 0) wrapped_gitaly_errors do gitaly_commit_client.diverging_commit_count(from, to, max_count: max_count) end @@ -732,18 +738,29 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + reachable_ref = + if source_repository == self + source_branch_name + else + # If a tmp ref was created before for a separate repo comparison (forks), + # we're able to short-circuit the tmp ref re-creation: + # 1. Take the SHA from the source repo + # 2. Read that in the current "target" repo + # 3. If that SHA is still known (readable), it means GC hasn't + # cleaned it up yet, so we can use it instead re-writing the tmp ref. + source_commit_id = source_repository.commit(source_branch_name)&.sha + commit(source_commit_id)&.sha if source_commit_id + end + + return compare(target_branch_name, reachable_ref, straight: straight) if reachable_ref + tmp_ref = "refs/tmp/#{SecureRandom.hex}" return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) - Gitlab::Git::Compare.new( - self, - target_branch_name, - tmp_ref, - straight: straight - ) + compare(target_branch_name, tmp_ref, straight: straight) ensure - delete_refs(tmp_ref) + delete_refs(tmp_ref) if tmp_ref end def write_ref(ref_path, ref, old_ref: nil) @@ -817,7 +834,8 @@ module Gitlab gitaly_repository_client.create_from_snapshot(url, auth) end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + # DEPRECATED: https://gitlab.com/gitlab-org/gitaly/issues/1628 + def rebase_deprecated(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) wrapped_gitaly_errors do gitaly_operation_client.user_rebase(user, rebase_id, branch: branch, @@ -827,6 +845,20 @@ module Gitlab end end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, &block) + wrapped_gitaly_errors do + gitaly_operation_client.rebase( + user, + rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch, + &block + ) + end + end + def rebase_in_progress?(rebase_id) wrapped_gitaly_errors do gitaly_repository_client.rebase_in_progress?(rebase_id) @@ -890,6 +922,12 @@ module Gitlab end end + def disconnect_alternates + wrapped_gitaly_errors do + gitaly_repository_client.disconnect_alternates + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository, @gl_project_path) end @@ -999,6 +1037,13 @@ module Gitlab private + def compare(base_ref, head_ref, straight:) + Gitlab::Git::Compare.new(self, + base_ref, + head_ref, + straight: straight) + end + def empty_diff_stats Gitlab::Git::DiffStatsCollection.new([]) end diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb index 2d1d8435cf3..9dd0ddfb44b 100644 --- a/lib/gitlab/git/repository_cleaner.rb +++ b/lib/gitlab/git/repository_cleaner.rb @@ -12,9 +12,9 @@ module Gitlab @repository = repository end - def apply_bfg_object_map(io) + def apply_bfg_object_map_stream(io, &blk) wrapped_gitaly_errors do - gitaly_cleanup_client.apply_bfg_object_map(io) + gitaly_cleanup_client.apply_bfg_object_map_stream(io, &blk) end end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index bb13d114d46..9c37bb01961 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -43,6 +43,8 @@ module Gitlab ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) end end + + ordered_entries end def rugged_populate_flat_path(repository, sha, path, entries) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index a0dd4a24363..c1bcd8e934a 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -86,9 +86,14 @@ module Gitlab end end - def pages(limit: 0, sort: nil, direction_desc: false) + def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) wrapped_gitaly_errors do - gitaly_get_all_pages(limit: limit, sort: sort, direction_desc: direction_desc) + gitaly_list_pages( + limit: limit, + sort: sort, + direction_desc: direction_desc, + load_content: load_content + ) end end @@ -168,10 +173,17 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end - def gitaly_get_all_pages(limit: 0, sort: nil, direction_desc: false) - gitaly_wiki_client.get_all_pages( - limit: limit, sort: sort, direction_desc: direction_desc - ).map do |wiki_page, version| + def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) + params = { limit: limit, sort: sort, direction_desc: direction_desc } + + gitaly_pages = + if load_content + gitaly_wiki_client.load_all_pages(params) + else + gitaly_wiki_client.list_all_pages(params) + end + + gitaly_pages.map do |wiki_page, version| Gitlab::Git::WikiPage.new(wiki_page, version) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index cb80ed64eff..4b626509008 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -85,7 +85,7 @@ module Gitlab check_push_access! end - ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages(cmd)) + success_result(cmd) end def guest_can_download_code? @@ -365,6 +365,10 @@ module Gitlab protected + def success_result(cmd) + ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages(cmd)) + end + def changes_list @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes) end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 3f13ebeb9d0..dfff6823689 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -5,12 +5,15 @@ module Gitlab module GitRefValidator extend self + + EXPANDED_PREFIXES = %w[refs/heads/ refs/remotes/].freeze + DISALLOWED_PREFIXES = %w[-].freeze + # Validates a given name against the git reference specification # # Returns true for a valid reference name, false otherwise def validate(ref_name) - not_allowed_prefixes = %w(refs/heads/ refs/remotes/ -) - return false if ref_name.start_with?(*not_allowed_prefixes) + return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) return false if ref_name == 'HEAD' begin @@ -19,5 +22,21 @@ module Gitlab return false end end + + def validate_merge_request_branch(ref_name) + return false if ref_name.start_with?(*DISALLOWED_PREFIXES) + + expanded_name = if ref_name.start_with?(*EXPANDED_PREFIXES) + ref_name + else + "refs/heads/#{ref_name}" + end + + begin + Rugged::Reference.valid_name?(expanded_name) + rescue ArgumentError + return false + end + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c432317eb24..05e06eec012 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,11 +26,15 @@ module Gitlab end end - PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze + # Server feature flags should use '_' to separate words. + SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE, 'delta_islands'].freeze + MUTEX = Mutex.new define_histogram :gitaly_controller_action_duration_seconds do @@ -219,6 +223,7 @@ module Gitlab metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id + metadata['gitaly-session-id'] = session_id if feature_enabled?(SERVER_FEATURE_CATFILE_CACHE) metadata.merge!(server_feature_flags) @@ -235,7 +240,9 @@ module Gitlab result end - SERVER_FEATURE_FLAGS = %w[].freeze + def self.session_id + Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid + end def self.server_feature_flags SERVER_FEATURE_FLAGS.map do |f| diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 6b8e58e6199..8ccefb00d20 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -55,13 +55,13 @@ module Gitlab def get_blobs(revision_paths, limit = -1) return [] if revision_paths.empty? - revision_paths.map! do |rev, path| + request_revision_paths = revision_paths.map do |rev, path| Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) end request = Gitaly::GetBlobsRequest.new( repository: @gitaly_repo, - revision_paths: revision_paths, + revision_paths: request_revision_paths, limit: limit ) diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index 3e8d6a773ca..a56bc35f6d7 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -12,25 +12,32 @@ module Gitlab @storage = repository.storage end - def apply_bfg_object_map(io) - first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo) + def apply_bfg_object_map_stream(io, &blk) + responses = GitalyClient.call( + storage, + :cleanup_service, + :apply_bfg_object_map_stream, + build_object_map_enum(io), + timeout: GitalyClient.no_timeout + ) + + responses.each(&blk) + end + + private - enum = Enumerator.new do |y| - y.yield first_request + def build_object_map_enum(io) + Enumerator.new do |y| + # First request. For simplicity, doesn't include any object map data + y << Gitaly::ApplyBfgObjectMapStreamRequest.new(repository: gitaly_repo) + # Now stream the BFG object map file to gitaly in chunks while data = io.read(RepositoryService::MAX_MSG_SIZE) - y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + y << Gitaly::ApplyBfgObjectMapStreamRequest.new(object_map: data) + break if io&.eof? end end - - GitalyClient.call( - storage, - :cleanup_service, - :apply_bfg_object_map, - enum, - timeout: GitalyClient.no_timeout - ) end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0d5debfcd01..d21b98d36ea 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -79,7 +79,7 @@ module Gitlab def tree_entry(ref, path, limit = nil) if Pathname.new(path).cleanpath.to_s.start_with?('../') - # The TreeEntry RPC should return an empty reponse in this case but in + # The TreeEntry RPC should return an empty response in this case but in # Gitaly 0.107.0 and earlier we get an exception instead. This early return # saves us a Gitaly roundtrip while also avoiding the exception. return @@ -174,7 +174,7 @@ module Gitlab response.each_with_object({}) do |gitaly_response, hsh| gitaly_response.commits.each do |commit_for_tree| - hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + hsh[commit_for_tree.path_bytes] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) end end end diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index ce1fb4d68ae..d7fac26bc13 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -33,6 +33,15 @@ module Gitlab GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, request, timeout: GitalyClient.fast_timeout) end + + def fetch(repository) + request = Gitaly::FetchIntoObjectPoolRequest.new( + object_pool: object_pool, + origin: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index b0f328ce3d4..e4a59ee3f9b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -197,6 +197,7 @@ module Gitlab start_repository: start_repository) end + # DEPRECATED: https://gitlab.com/gitlab-org/gitaly/issues/1628 def user_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) request = Gitaly::UserRebaseRequest.new( repository: @gitaly_repo, @@ -225,6 +226,49 @@ module Gitlab end end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + request_enum = QueueEnumerator.new + rebase_sha = nil + + response_enum = GitalyClient.call( + @repository.storage, + :operation_service, + :user_rebase_confirmable, + request_enum.each, + remote_storage: remote_repository.storage + ) + + # First request + request_enum.push( + Gitaly::UserRebaseConfirmableRequest.new( + header: Gitaly::UserRebaseConfirmableRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + rebase_id: rebase_id.to_s, + branch: encode_binary(branch), + branch_sha: branch_sha, + remote_repository: remote_repository.gitaly_repository, + remote_branch: encode_binary(remote_branch) + ) + ) + ) + + perform_next_gitaly_rebase_request(response_enum) do |response| + rebase_sha = response.rebase_sha + end + + yield rebase_sha + + # Second request confirms with gitaly to finalize the rebase + request_enum.push(Gitaly::UserRebaseConfirmableRequest.new(apply: true)) + + perform_next_gitaly_rebase_request(response_enum) + + rebase_sha + ensure + request_enum.close + end + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, @@ -346,6 +390,20 @@ module Gitlab private + def perform_next_gitaly_rebase_request(response_enum) + response = response_enum.next + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.git_error.present? + raise Gitlab::Git::Repository::GitError, response.git_error + end + + yield response if block_given? + + response + end + def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 6f6698607d9..b7d509dfa48 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -239,6 +239,12 @@ module Gitlab messages end + def pack_refs + request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :ref_service, :pack_refs, request) + end + private def consume_refs_response(response) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 74aae4a8e97..68b17e86608 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -331,6 +331,14 @@ module Gitlab search_results_from_response(response) end + def disconnect_alternates + request = Gitaly::DisconnectGitAlternatesRequest.new( + repository: @gitaly_repo + ) + + GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request) + end + private def search_results_from_response(gitaly_response) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index e036cdcd800..ce9faad825c 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -87,7 +87,27 @@ module Gitlab wiki_page_from_iterator(response) end - def get_all_pages(limit: 0, sort: nil, direction_desc: false) + def list_all_pages(limit: 0, sort: nil, direction_desc: false) + sort_value = Gitaly::WikiListPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) + + params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } + params[:sort] = sort_value if sort_value + + request = Gitaly::WikiListPagesRequest.new(params) + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_list_pages, request, timeout: GitalyClient.medium_timeout) + stream.each_with_object([]) do |message, pages| + page = message.page + + next unless page + + wiki_page = GitalyClient::WikiPage.new(page.to_h) + version = new_wiki_page_version(page.version) + + pages << [wiki_page, version] + end + end + + def load_all_pages(limit: 0, sort: nil, direction_desc: false) sort_value = Gitaly::WikiGetAllPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } @@ -95,6 +115,7 @@ module Gitlab request = Gitaly::WikiGetAllPagesRequest.new(params) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) + pages = [] loop do diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 656d46b6a7d..a468f6d8821 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -53,6 +53,7 @@ module Gitlab description: description, milestone_id: milestone_finder.id_for(issue), state: issue.state, + state_id: ::Issue.available_states[issue.state], created_at: issue.created_at, updated_at: issue.updated_at } diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 1b293ddc7c7..377e873d24d 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -55,6 +55,7 @@ module Gitlab source_branch: pull_request.formatted_source_branch, target_branch: pull_request.target_branch, state: pull_request.state, + state_id: ::MergeRequest.available_states[pull_request.state], milestone_id: milestone_finder.id_for(pull_request), author_id: author_id, assignee_id: user_finder.assignee_id_for(pull_request), diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index be1334ca98a..d336b1ba797 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 070e3b2db8d..5b98ce7d5ed 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index a56ca1e39e7..04dabe423e8 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true module Gitlab - module GlRepository + class GlRepository + include Singleton + PROJECT = RepoType.new( name: :project, access_checker_class: Gitlab::GitAccess, @@ -19,7 +21,7 @@ module Gitlab }.freeze def self.types - TYPES + instance.types end def self.parse(gl_repository) @@ -39,5 +41,11 @@ module Gitlab def self.default_type PROJECT end + + def types + TYPES + end + + private_class_method :instance end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 7abe6c29a25..19915980d7f 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module GlRepository + class GlRepository class RepoType attr_reader :name, :access_checker_class, diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index e00309e7946..582c3065189 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -15,7 +15,12 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled + + if Gitlab::CurrentSettings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn + gon.sentry_environment = Gitlab.config.sentry.environment + end + gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab.revision gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 03d6aabb0e3..619ce100421 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -48,7 +48,7 @@ module Gitlab end def authorize_against(parent_typed_object, resolved_type) - if built_in_type? + if scalar_type? # The field is a built-in/scalar type, or a list of scalars # authorize using the parent's object parent_typed_object.object @@ -108,8 +108,8 @@ module Gitlab type.unwrap end - def built_in_type? - GraphQL::Schema::BUILT_IN_TYPES.has_value?(node_type_for_basic_connection(@field.type)) + def scalar_type? + node_type_for_basic_connection(@field.type).kind.scalar? end end end diff --git a/lib/gitlab/graphql/tracing.rb b/lib/gitlab/graphql/generic_tracing.rb index 6b505e4262b..936b22d5afa 100644 --- a/lib/gitlab/graphql/tracing.rb +++ b/lib/gitlab/graphql/generic_tracing.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true +# This class is used as a hook to observe graphql runtime events. From this +# hook both gitlab metrics and opentracking measurements are generated + module Gitlab module Graphql - class Tracing < GraphQL::Tracing::PlatformTracing + class GenericTracing < GraphQL::Tracing::PlatformTracing self.platform_keys = { 'lex' => 'graphql.lex', 'parse' => 'graphql.parse', @@ -21,17 +24,30 @@ module Gitlab end def platform_trace(platform_key, key, data, &block) + tags = { platform_key: platform_key, key: key } start = Gitlab::Metrics::System.monotonic_time - yield + with_labkit_tracing(tags, &block) ensure duration = Gitlab::Metrics::System.monotonic_time - start - graphql_duration_seconds.observe({ platform_key: platform_key, key: key }, duration) + graphql_duration_seconds.observe(tags, duration) end private + def with_labkit_tracing(tags, &block) + return yield unless Labkit::Tracing.enabled? + + name = "#{tags[:platform_key]}.#{tags[:key]}" + span_tags = { + 'component' => 'web', + 'span.kind' => 'server' + }.merge(tags.stringify_keys) + + Labkit::Tracing.with_tracing(operation_name: name, tags: span_tags, &block) + end + def graphql_duration_seconds @graphql_duration_seconds ||= Gitlab::Metrics.histogram( :graphql_duration_seconds, diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 7255293b194..334642f252e 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -2,6 +2,8 @@ module Gitlab class GroupSearchResults < SearchResults + attr_reader :group + def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) @@ -26,5 +28,9 @@ module Gitlab .where(id: groups.select('members.user_id')) end # rubocop:enable CodeReuse/ActiveRecord + + def issuable_params + super.merge(group_id: group.id) + end end end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index 62a5216d159..184083de2bc 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index d32a6980eb8..4586b1d94a7 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index bcd9e2be35f..55ed7e7e837 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -11,7 +11,7 @@ module Gitlab include HTTParty # rubocop:disable Gitlab/HTTParty - connection_adapter ProxyHTTPConnectionAdapter + connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) super diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index a64cb47e77e..41eab3658bc 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -10,17 +10,19 @@ # # This option will take precedence over the global setting. module Gitlab - class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter + class HTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - unless allow_local_requests? - begin - Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) - rescue Gitlab::UrlBlocker::BlockedUrlError => e - raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" - end + begin + @uri, hostname = Gitlab::UrlBlocker.validate!(uri, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection?) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" end - super + super.tap do |http| + http.hostname_override = hostname if hostname + end end private @@ -29,6 +31,12 @@ module Gitlab options.fetch(:allow_local_requests, allow_settings_local_requests?) end + def dns_rebind_protection? + return false if Gitlab.http_proxy_env? + + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end + def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index fcf6a25ab00..acb7f225b17 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -30,10 +30,7 @@ module Gitlab def handle_response_error(response) unless response.success? - error_code = response.dig('Error', 'Code') || response.code - error_message = response.dig('Error', 'Message') || response.message - - raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}") end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ce268793128..7bbcb53f016 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -75,6 +75,7 @@ project_tree: - :project_badges - :ci_cd_settings - :error_tracking_setting + - :metrics_setting # Only include the following attributes for the models specified. included_attributes: @@ -175,6 +176,8 @@ excluded_attributes: - :enabled methods: + notes: + - :type labels: - :type label: diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6be95a16513..5ab5059165e 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -59,7 +59,11 @@ module Gitlab end def member_hash(member) - parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + parsed_hash(member).merge( + 'source_id' => @project.id, + 'importing' => true, + 'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min + ) end def parsed_hash(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 51001750a6c..20caadb89c0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -129,7 +129,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level - level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level + level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level { 'visibility_level' => level } end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 61a1aa6da5a..e1e70a008d9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -25,7 +25,8 @@ module Gitlab metrics: 'MergeRequest::Metrics', ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link' }.freeze + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze diff --git a/lib/gitlab/kubernetes/errors.rb b/lib/gitlab/kubernetes/errors.rb new file mode 100644 index 00000000000..81bf636eef7 --- /dev/null +++ b/lib/gitlab/kubernetes/errors.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Errors + CONNECTION = [ + SocketError, + OpenSSL::SSL::SSLError, + Errno::ECONNRESET, + Errno::ENETUNREACH, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Net::OpenTimeout, + Net::ReadTimeout, + IPAddr::InvalidAddressError + ].freeze + + AUTHENTICATION = [ + OpenSSL::X509::CertificateError + ].freeze + end + end +end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 7dfd9ed4f35..ff1dadf9247 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -22,6 +22,13 @@ module Gitlab alias_method :update, :install + def uninstall(command) + namespace.ensure_exists! + + delete_pod!(command.pod_name) + kubeclient.create_pod(command.pod_resource) + end + ## # Returns Pod phase # diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb new file mode 100644 index 00000000000..6a7f5e965c5 --- /dev/null +++ b/lib/gitlab/lets_encrypt/challenge.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module LetsEncrypt + class Challenge + def initialize(acme_challenge) + @acme_challenge = acme_challenge + end + + delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge + + private + + attr_reader :acme_challenge + end + end +end diff --git a/lib/gitlab/lets_encrypt/client.rb b/lib/gitlab/lets_encrypt/client.rb new file mode 100644 index 00000000000..5501f7981ec --- /dev/null +++ b/lib/gitlab/lets_encrypt/client.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module LetsEncrypt + class Client + PRODUCTION_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory' + STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' + + def new_order(domain_name) + ensure_account + + acme_order = acme_client.new_order(identifiers: [domain_name]) + + ::Gitlab::LetsEncrypt::Order.new(acme_order) + end + + def load_order(url) + ensure_account + + # rubocop: disable CodeReuse/ActiveRecord + ::Gitlab::LetsEncrypt::Order.new(acme_client.order(url: url)) + # rubocop: enable CodeReuse/ActiveRecord + end + + def load_challenge(url) + ensure_account + + ::Gitlab::LetsEncrypt::Challenge.new(acme_client.challenge(url: url)) + end + + def terms_of_service_url + acme_client.terms_of_service + end + + def enabled? + return false unless Feature.enabled?(:pages_auto_ssl) + + Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted + end + + private + + def acme_client + @acme_client ||= ::Acme::Client.new(private_key: private_key, directory: acme_api_directory_url) + end + + def private_key + @private_key ||= OpenSSL::PKey.read(Gitlab::CurrentSettings.lets_encrypt_private_key) + end + + def admin_email + Gitlab::CurrentSettings.lets_encrypt_notification_email + end + + def contact + "mailto:#{admin_email}" + end + + def ensure_account + raise 'Acme integration is disabled' unless enabled? + + @acme_account ||= acme_client.new_account(contact: contact, terms_of_service_agreed: true) + end + + def acme_api_directory_url + if Rails.env.production? + PRODUCTION_DIRECTORY_URL + else + STAGING_DIRECTORY_URL + end + end + end + end +end diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb new file mode 100644 index 00000000000..5109b5e9843 --- /dev/null +++ b/lib/gitlab/lets_encrypt/order.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module LetsEncrypt + class Order + def initialize(acme_order) + @acme_order = acme_order + end + + def new_challenge + authorization = @acme_order.authorizations.first + challenge = authorization.http + ::Gitlab::LetsEncrypt::Challenge.new(challenge) + end + + delegate :url, :status, to: :acme_order + + private + + attr_reader :acme_order + end + end +end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 31e6fc9d8c7..124e34562c1 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -35,8 +35,7 @@ module Gitlab end def token_valid?(token_to_check) - HMACToken.new(actor).token_valid?(token_to_check) || - LegacyRedisDeviseToken.new(actor).token_valid?(token_to_check) + HMACToken.new(actor).token_valid?(token_to_check) end def deploy_key_pushable?(project) @@ -103,44 +102,5 @@ module Gitlab Settings.attr_encrypted_db_key_base.first(16) end end - - # TODO: LegacyRedisDeviseToken and references need to be removed after - # next released milestone - # - class LegacyRedisDeviseToken - TOKEN_LENGTH = 50 - DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins - - def initialize(actor) - @actor = actor - end - - def token_valid?(token_to_check) - Devise.secure_compare(stored_token, token_to_check) - end - - def stored_token - Gitlab::Redis::SharedState.with { |redis| redis.get(state_key) } - end - - # This method exists purely to facilitate legacy testing to ensure the - # same redis key is used. - # - def store_new_token(expiry_time_in_ms = DEFAULT_EXPIRY_TIME) - Gitlab::Redis::SharedState.with do |redis| - new_token = Devise.friendly_token(TOKEN_LENGTH) - redis.set(state_key, new_token, px: expiry_time_in_ms) - new_token - end - end - - private - - attr_reader :actor - - def state_key - "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" - end - end end end diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb new file mode 100644 index 00000000000..94aabd0466c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/base_service.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +module Gitlab + module Metrics + module Dashboard + class BaseService < ::BaseService + DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError + + def get_dashboard + return error("#{dashboard_path} could not be found.", :not_found) unless path_available? + + success(dashboard: process_dashboard) + rescue DASHBOARD_LAYOUT_ERROR => e + error(e.message, :unprocessable_entity) + end + + # Summary of all known dashboards for the service. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def all_dashboard_paths(_project) + raise NotImplementedError + end + + private + + # Returns a new dashboard Hash, supplemented with DB info + def process_dashboard + Gitlab::Metrics::Dashboard::Processor + .new(project, params[:environment], raw_dashboard) + .process(insert_project_metrics: insert_project_metrics?) + end + + # @return [String] Relative filepath of the dashboard yml + def dashboard_path + params[:dashboard_path] + end + + # Returns an un-processed dashboard from the cache. + def raw_dashboard + Rails.cache.fetch(cache_key) { get_raw_dashboard } + end + + # @return [Hash] an unmodified dashboard + def get_raw_dashboard + raise NotImplementedError + end + + # @return [String] + def cache_key + raise NotImplementedError + end + + # Determines whether custom metrics should be included + # in the processed output. + def insert_project_metrics? + false + end + + # Checks if dashboard path exists or should be rejected + # as a result of file-changes to the project repository. + # @return [Boolean] + def path_available? + available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project) + + available_paths.any? do |path_params| + path_params[:path] == dashboard_path + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb new file mode 100644 index 00000000000..4a41590f000 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# Returns DB-supplmented dashboard info for determining +# the layout of UI. Intended entry-point for the Metrics::Dashboard +# module. +module Gitlab + module Metrics + module Dashboard + class Finder + class << self + # Returns a formatted dashboard packed with DB info. + # @return [Hash] + def find(project, user, environment, dashboard_path = nil) + service = system_dashboard?(dashboard_path) ? system_service : project_service + + service + .new(project, user, environment: environment, dashboard_path: dashboard_path) + .get_dashboard + end + + # Summary of all known dashboards. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def find_all_paths(project) + project.repository.metrics_dashboard_paths + end + + # Summary of all known dashboards. Used to populate repo cache. + # Prefer #find_all_paths. + def find_all_paths_from_source(project) + system_service.all_dashboard_paths(project) + .+ project_service.all_dashboard_paths(project) + end + + private + + def system_service + Gitlab::Metrics::Dashboard::SystemDashboardService + end + + def project_service + Gitlab::Metrics::Dashboard::ProjectDashboardService + end + + def system_dashboard?(filepath) + !filepath || system_service.system_dashboard?(filepath) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb new file mode 100644 index 00000000000..dd986020693 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + # Responsible for processesing a dashboard hash, inserting + # relevant DB records & sorting for proper rendering in + # the UI. These includes shared metric info, custom metrics + # info, and alerts (only in EE). + class Processor + SYSTEM_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::ProjectMetricsInserter, + Stages::Sorter + ].freeze + + PROJECT_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::Sorter + ].freeze + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Returns a new dashboard hash with the results of + # running transforms on the dashboard. + def process(insert_project_metrics:) + @dashboard.deep_symbolize_keys.tap do |dashboard| + sequence(insert_project_metrics).each do |stage| + stage.new(@project, @environment, dashboard).transform! + end + end + end + + private + + def sequence(insert_project_metrics) + insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb new file mode 100644 index 00000000000..fdffd067c93 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService + DASHBOARD_ROOT = ".gitlab/dashboards" + + class << self + def all_dashboard_paths(project) + file_finder(project) + .list_files_for(DASHBOARD_ROOT) + .map do |filepath| + Rails.cache.delete(cache_key(project.id, filepath)) + + { path: filepath, default: false } + end + end + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') + end + + def cache_key(id, dashboard_path) + "project_#{id}_metrics_dashboard_#{dashboard_path}" + end + end + + private + + # Searches the project repo for a custom-defined dashboard. + def get_raw_dashboard + yml = self.class.file_finder(project).read(dashboard_path) + + YAML.safe_load(yml) + end + + def cache_key + self.class.cache_key(project.id, dashboard_path) + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb new file mode 100644 index 00000000000..a6d1f974556 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class BaseStage + DashboardLayoutError = Class.new(StandardError) + + DEFAULT_PANEL_TYPE = 'area-chart' + + attr_reader :project, :environment, :dashboard + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Entry-point to the stage + def transform! + raise NotImplementedError + end + + protected + + def missing_panel_groups! + raise DashboardLayoutError.new('Top-level key :panel_groups must be an array') + end + + def missing_panels! + raise DashboardLayoutError.new('Each "panel_group" must define an array :panels') + end + + def missing_metrics! + raise DashboardLayoutError.new('Each "panel" must define an array :metrics') + end + + def for_metrics + missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) + + dashboard[:panel_groups].each do |panel_group| + missing_panels! unless panel_group[:panels].is_a?(Array) + + panel_group[:panels].each do |panel| + missing_metrics! unless panel[:metrics].is_a?(Array) + + panel[:metrics].each do |metric| + yield metric + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb new file mode 100644 index 00000000000..188912bedb4 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class CommonMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, + # includes the record's id in the dashboard config. + def transform! + common_metrics = ::PrometheusMetric.common + + for_metrics do |metric| + metric_record = common_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb new file mode 100644 index 00000000000..221610a14d1 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class ProjectMetricsInserter < BaseStage + # Inserts project-specific metrics into the dashboard + # config. If there are no project-specific metrics, + # this will have no effect. + def transform! + project.prometheus_metrics.each do |project_metric| + group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) + panel = find_or_create_panel(group[:panels], project_metric) + find_or_create_metric(panel[:metrics], project_metric) + end + end + + private + + # Looks for a panel_group corresponding to the + # provided metric object. If unavailable, inserts one. + # @param panel_groups [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel_group(panel_groups, metric) + panel_group = find_panel_group(panel_groups, metric) + return panel_group if panel_group + + panel_group = new_panel_group(metric) + panel_groups << panel_group + + panel_group + end + + # Looks for a panel corresponding to the provided + # metric object. If unavailable, inserts one. + # @param panels [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel(panels, metric) + panel = find_panel(panels, metric) + return panel if panel + + panel = new_panel(metric) + panels << panel + + panel + end + + # Looks for a metric corresponding to the provided + # metric object. If unavailable, inserts one. + # @param metrics [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_metric(metrics, metric) + target_metric = find_metric(metrics, metric) + return target_metric if target_metric + + target_metric = new_metric(metric) + metrics << target_metric + + target_metric + end + + def find_panel_group(panel_groups, metric) + return unless panel_groups + + panel_groups.find { |group| group[:group] == metric.group_title } + end + + def find_panel(panels, metric) + return unless panels + + panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] + panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } + end + + def find_metric(metrics, metric) + return unless metrics + + metrics.find { |m| m[:id] == metric.identifier } + end + + def new_panel_group(metric) + { + group: metric.group_title, + priority: metric.priority, + panels: [] + } + end + + def new_panel(metric) + { + type: DEFAULT_PANEL_TYPE, + title: metric.title, + y_label: metric.y_label, + metrics: [] + } + end + + def new_metric(metric) + metric.queries.first.merge(metric_id: metric.id) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb new file mode 100644 index 00000000000..ba5aa78059c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class Sorter < BaseStage + def transform! + missing_panel_groups! unless dashboard[:panel_groups].is_a? Array + + sort_groups! + sort_panels! + end + + private + + # Sorts the groups in the dashboard by the :priority key + def sort_groups! + dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i } + end + + # Sorts the panels in the dashboard by the :weight key + def sort_panels! + dashboard[:panel_groups].each do |group| + missing_panels! unless group[:panels].is_a? Array + + group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i } + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb new file mode 100644 index 00000000000..67509ed4230 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Fetches the system metrics dashboard and formats the output. +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService + SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' + + class << self + def all_dashboard_paths(_project) + [{ + path: SYSTEM_DASHBOARD_PATH, + default: true + }] + end + + def system_dashboard?(filepath) + filepath == SYSTEM_DASHBOARD_PATH + end + end + + private + + def dashboard_path + SYSTEM_DASHBOARD_PATH + end + + # Returns the base metrics shipped with every GitLab service. + def get_raw_dashboard + yml = File.read(Rails.root.join(dashboard_path)) + + YAML.safe_load(yml) + end + + def cache_key + "metrics_dashboard_#{dashboard_path}" + end + + def insert_project_metrics? + true + end + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 9e4d70a71ff..30f181542be 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric - JITTER_RANGE = 0.000001..0.001 + JITTER_RANGE = (0.000001..0.001).freeze attr_reader :series, :values, :tags, :type diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 83c52a6c6e0..8e17073abab 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -3,7 +3,7 @@ module Gitlab module Middleware class ReadOnly - API_VERSIONS = (3..4) + API_VERSIONS = (3..4).freeze def self.internal_routes @internal_routes ||= diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb index 849cf8f759b..0719fb2e8c6 100644 --- a/lib/gitlab/middleware/release_env.rb +++ b/lib/gitlab/middleware/release_env.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/namespaced_session_store.rb b/lib/gitlab/namespaced_session_store.rb new file mode 100644 index 00000000000..34520078bfb --- /dev/null +++ b/lib/gitlab/namespaced_session_store.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + class NamespacedSessionStore + delegate :[], :[]=, to: :store + + def initialize(key) + @key = key + end + + def initiated? + !Session.current.nil? + end + + def store + return unless Session.current + + Session.current[@key] ||= {} + Session.current[@key] + end + end +end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index e0ac9eec1f2..2a2083ebae0 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -36,12 +36,25 @@ module Gitlab hash_arguments = provider['args'].merge(provider_defaults(provider)) # A Hash from the configuration will be passed as is. - provider_arguments << hash_arguments.symbolize_keys + provider_arguments << normalize_hash_arguments(hash_arguments) end provider_arguments end + def normalize_hash_arguments(args) + args.symbolize_keys! + + # Rails 5.1 deprecated the use of string names in the middleware + # (https://github.com/rails/rails/commit/83b767ce), so we need to + # pass in the actual class to Devise. + if args[:strategy_class].is_a?(String) + args[:strategy_class] = args[:strategy_class].constantize + end + + args + end + def provider_defaults(provider) case provider['name'] when 'cas3' diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a07b1246bee..aa2c1ac9cef 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -58,6 +58,7 @@ module Gitlab uploads users v2 + visual-review-toolbar.js ].freeze # This list should contain all words following `/*namespace_id/:project_id` in diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index ac392432427..16c16aa0265 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -26,11 +26,7 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if Gitlab::SafeRequestStore.store[:peek_enabled] - # data[:cached] is only available starting from Rails 5.1.0 - # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 - # Before that, data[:name] was set to 'CACHE' - # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 - unless data.fetch(:cached, data[:name] == 'CACHE') + unless data[:cached] track_query(data[:sql].strip, data[:binds], start, finish) end end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 28ed587f5c7..890228e5e78 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -73,7 +73,7 @@ module Gitlab result = with_custom_logger(logger) do with_user(user) do - RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 58f06b6708c..0f3b97e2317 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -138,6 +138,12 @@ module Gitlab project end + def filter_milestones_by_project(milestones) + return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) + + milestones.where(project_id: project.id) # rubocop: disable CodeReuse/ActiveRecord + end + def repository_project_ref @repository_project_ref ||= repository_ref || project.default_branch end @@ -145,5 +151,9 @@ module Gitlab def repository_wiki_ref @repository_wiki_ref ||= repository_ref || project.wiki.default_branch end + + def issuable_params + super.merge(project_id: project.id) + end end end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 1cc85d4b4a6..9cc21129547 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -4,9 +4,12 @@ module Gitlab module Prometheus module QueryVariables def self.call(environment) + deployment_platform = environment.deployment_platform + namespace = deployment_platform&.kubernetes_namespace_for(environment.project) || '' + { ci_environment_slug: environment.slug, - kube_namespace: environment.deployment_platform&.actual_namespace || '', + kube_namespace: namespace, environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 810aba436cc..3137676ba4b 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -15,7 +15,7 @@ module Gitlab mr: :merge_request }).freeze - OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/ + OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze attr_reader :options diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 1cd158db2b0..e1579cfddc0 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -102,7 +102,7 @@ module Gitlab @updates[:milestone_id] = nil end - desc _('Copy labels and milestone from other issue or merge request') + desc _('Copy labels and milestone from other issue or merge request in this project') explanation do |source_issuable| _("Copy labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index f5176376a60..4a62e83e8e9 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -11,7 +11,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}.freeze def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 3b05f181ed2..84885be9bda 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -45,7 +45,7 @@ module Gitlab def self.redirect_legacy_paths(router, *paths) build_redirect_path = lambda do |request, _params, path| - # Only replace the last occurence of `path`. + # Only replace the last occurrence of `path`. # # `request.fullpath` includes the querystring new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index 0d4e6be2129..98f78c5e74b 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -9,7 +9,7 @@ module Gitlab class Scrubber < Loofah::Scrubber # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes - DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u + DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u.freeze def scrub(node) unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 8988b9ad7be..7c1e6b1baff 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -2,6 +2,8 @@ module Gitlab class SearchResults + COUNT_LIMIT = 1001 + attr_reader :current_user, :query, :per_page # Limit search results by passed projects @@ -25,29 +27,26 @@ module Gitlab def objects(scope, page = nil, without_count = true) collection = case scope when 'projects' - projects.page(page).per(per_page) + projects when 'issues' - issues.page(page).per(per_page) + issues when 'merge_requests' - merge_requests.page(page).per(per_page) + merge_requests when 'milestones' - milestones.page(page).per(per_page) + milestones when 'users' - users.page(page).per(per_page) + users else - Kaminari.paginate_array([]).page(page).per(per_page) - end + Kaminari.paginate_array([]) + end.page(page).per(per_page) without_count ? collection.without_count : collection end - # rubocop: disable CodeReuse/ActiveRecord def limited_projects_count - @limited_projects_count ||= projects.limit(count_limit).count + @limited_projects_count ||= limited_count(projects) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_issues_count return @limited_issues_count if @limited_issues_count @@ -56,35 +55,28 @@ module Gitlab # and confidential issues user has access to, is too complex. # It's faster to try to fetch all public issues first, then only # if necessary try to fetch all issues. - sum = issues(public_only: true).limit(count_limit).count - @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum + sum = limited_count(issues(public_only: true)) + @limited_issues_count = sum < count_limit ? limited_count(issues) : sum end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_merge_requests_count - @limited_merge_requests_count ||= merge_requests.limit(count_limit).count + @limited_merge_requests_count ||= limited_count(merge_requests) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_milestones_count - @limited_milestones_count ||= milestones.limit(count_limit).count + @limited_milestones_count ||= limited_count(milestones) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop:disable CodeReuse/ActiveRecord def limited_users_count - @limited_users_count ||= users.limit(count_limit).count + @limited_users_count ||= limited_count(users) end - # rubocop:enable CodeReuse/ActiveRecord def single_commit_result? false end def count_limit - 1001 + COUNT_LIMIT end def users @@ -99,58 +91,82 @@ module Gitlab limit_projects.search(query) end - # rubocop: disable CodeReuse/ActiveRecord def issues(finder_params = {}) - issues = IssuesFinder.new(current_user, finder_params).execute + issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute + unless default_project_filter - issues = issues.where(project_id: project_ids_relation) + issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - issues = - if query =~ /#(\d+)\z/ - issues.where(iid: $1) - else - issues.full_search(query) - end - - issues.reorder('updated_at DESC') + issues end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def milestones - milestones = Milestone.where(project_id: project_ids_relation) - milestones = milestones.search(query) + milestones = Milestone.search(query) + + milestones = filter_milestones_by_project(milestones) + milestones.reorder('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def merge_requests - merge_requests = MergeRequestsFinder.new(current_user).execute + merge_requests = MergeRequestsFinder.new(current_user, issuable_params).execute + unless default_project_filter merge_requests = merge_requests.in_projects(project_ids_relation) end - merge_requests = - if query =~ /[#!](\d+)\z/ - merge_requests.where(iid: $1) - else - merge_requests.full_search(query) - end - - merge_requests.reorder('updated_at DESC') + merge_requests end - # rubocop: enable CodeReuse/ActiveRecord def default_scope 'projects' end + # Filter milestones by authorized projects. + # For performance reasons project_id is being plucked + # to be used on a smaller query. + # + # rubocop: disable CodeReuse/ActiveRecord + def filter_milestones_by_project(milestones) + project_ids = + milestones.where(project_id: project_ids_relation) + .select(:project_id).distinct + .pluck(:project_id) + + return Milestone.none if project_ids.nil? + + authorized_project_ids_relation = + Project.where(id: project_ids).ids_with_milestone_available_for(current_user) + + milestones.where(project_id: authorized_project_ids_relation) + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) end # rubocop: enable CodeReuse/ActiveRecord + + def issuable_params + {}.tap do |params| + params[:sort] = 'updated_desc' + + if query =~ /#(\d+)\z/ + params[:iids] = $1 + else + params[:search] = query + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def limited_count(relation) + relation.reorder(nil).limit(count_limit).size + end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 356e6445e0e..72c44114001 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -10,7 +10,7 @@ module Gitlab def self.context(current_user = nil) return unless enabled? - Raven.tags_context(locale: I18n.locale) + Raven.tags_context(default_tags) if current_user Raven.user_context( @@ -44,16 +44,19 @@ module Gitlab extra[:issue_url] = issue_url if issue_url context # Make sure we've set everything we know in the context - tags = { - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id - } - - Raven.capture_exception(exception, tags: tags, extra: extra) + Raven.capture_exception(exception, tags: default_tags, extra: extra) end end def self.should_raise_for_dev? Rails.env.development? || Rails.env.test? end + + def self.default_tags + { + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id, + locale: I18n.locale + } + end end end diff --git a/lib/gitlab/session.rb b/lib/gitlab/session.rb new file mode 100644 index 00000000000..7487ba04a6d --- /dev/null +++ b/lib/gitlab/session.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + class Session + STORE_KEY = :session_storage + + class << self + def with_session(session) + old = self.current + self.current = session + yield + ensure + self.current = old + end + + def current + Thread.current[STORE_KEY] + end + + protected + + def current=(value) + Thread.current[STORE_KEY] = value + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 747cb0f9142..f7b08d58e49 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -4,9 +4,9 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = %r{text/html|application/json}i + CONTENT_TYPES = %r{text/html|application/json}i.freeze - IGNORE_PATHS = %r{^/sherlock} + IGNORE_PATHS = %r{^/sherlock}.freeze def initialize(app) @app = app diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 11561eec32a..159ce27e702 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -15,7 +15,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix.freeze # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index fb303e3fb0c..c102fa14cfc 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -7,7 +7,7 @@ module Gitlab module SidekiqConfig QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze - # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. def self.worker_queues(rails_path = Rails.root.to_s) @worker_queues ||= {} @@ -19,7 +19,7 @@ module Gitlab end end - # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. def self.expand_queues(queues, all_queues = self.worker_queues) return [] if queues.empty? diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 607c9c8dec1..a66a2e0726b 100644 --- a/lib/gitlab/slash_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index b698391c8bd..fd108b4c124 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do def fuzzy_search(query, columns) diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 641ba70ef83..9a8df719827 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -8,38 +8,68 @@ module Gitlab BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, ports: [], schemes: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false) - return true if url.nil? + # Validates the given url according to the constraints specified by arguments. + # + # ports - Raises error if the given URL port does is not between given ports. + # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is true. + # allow_local_network - Raises error if URL resolves to a link-local address and argument is true. + # ascii_only - Raises error if URL has unicode characters and argument is true. + # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. + # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # + # Returns an array with [<uri>, <original-hostname>]. + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/ParameterLists + def validate!( + url, + ports: [], + schemes: [], + allow_localhost: false, + allow_local_network: true, + ascii_only: false, + enforce_user: false, + enforce_sanitization: false, + dns_rebind_protection: true) + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/ParameterLists + + return [nil, nil] if url.nil? # Param url can be a string, URI or Addressable::URI uri = parse_url(url) validate_html_tags!(uri) if enforce_sanitization - # Allow imports from the GitLab instance itself but only from the configured ports - return true if internal?(uri) - + hostname = uri.hostname port = get_port(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(uri.hostname) - validate_unicode_restriction!(uri) if ascii_only + + unless internal?(uri) + validate_scheme!(uri.scheme, schemes) + validate_port!(port, ports) if ports.any? + validate_user!(uri.user) if enforce_user + validate_hostname!(hostname) + validate_unicode_restriction!(uri) if ascii_only + end begin - addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr| + addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError - return true + return [uri, nil] end + protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + + # Allow url from the GitLab instance itself but only for the configured hostname and ports + return protected_uri_with_hostname if internal?(uri) + validate_localhost!(addrs_info) unless allow_localhost validate_loopback!(addrs_info) unless allow_localhost validate_local_network!(addrs_info) unless allow_local_network validate_link_local!(addrs_info) unless allow_local_network - true + protected_uri_with_hostname end def blocked_url?(*args) @@ -52,6 +82,25 @@ module Gitlab private + # Returns the given URI with IP address as hostname and the original hostname respectively + # in an Array. + # + # It checks whether the resolved IP address matches with the hostname. If not, it changes + # the hostname to the resolved IP address. + # + # The original hostname is used to validate the SSL, given in that scenario + # we'll be making the request to the IP address, instead of using the hostname. + def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + address = addrs_info.first + ip_address = address&.ip_address + + return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname + + uri = uri.dup + uri.hostname = ip_address + [uri, hostname] + end + def get_port(uri) uri.port || uri.default_port end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f86d599e4cb..42cf1ec1f0e 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -30,6 +30,10 @@ module Gitlab snippet_url(object) when Milestone milestone_url(object) + when ::Ci::Build + project_job_url(object.project, object) + when User + user_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 880712de5fe..215454fe63c 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -47,6 +47,10 @@ module Gitlab @credentials ||= { user: @url.user.presence, password: @url.password.presence } end + def user + credentials[:user] + end + def full_url @full_url ||= generate_full_url.to_s end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 08156d7ffa6..9aa2e972adf 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -26,7 +26,7 @@ module Gitlab uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, - installation_type: Gitlab::INSTALLATION_TYPE, + installation_type: installation_type, active_user_count: count(User.active), recorded_at: Time.now, edition: 'CE' @@ -81,6 +81,7 @@ module Gitlab milestone_lists: count(List.milestone), milestones: count(Milestone), pages_domains: count(PagesDomain), + pool_repositories: count(PoolRepository), projects: count(Project), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), @@ -190,6 +191,14 @@ module Gitlab result[key] = approx_counts[model] || -1 end end + + def installation_type + if Rails.env.production? + Gitlab::INSTALLATION_TYPE + else + "gitlab-development-kit" + end + end end end end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index f0557f6ad68..ede60c9ab1d 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -7,7 +7,7 @@ module Gitlab class UserExtractor # Not using `Devise.email_regexp` to filter out any chars that an email # does not end with and not pinning the email to a start of end of a string. - EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/.freeze USERNAME_REGEXP = User.reference_pattern def initialize(text) diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb deleted file mode 100644 index 2e98227a05e..00000000000 --- a/lib/haml_lint/inline_javascript.rb +++ /dev/null @@ -1,26 +0,0 @@ -# rubocop:disable Naming/FileName -# frozen_string_literal: true - -unless Rails.env.production? - require_dependency 'haml_lint/haml_visitor' - require_dependency 'haml_lint/linter' - require_dependency 'haml_lint/linter_registry' - - module HamlLint - class Linter::InlineJavaScript < Linter - include LinterRegistry - - def visit_filter(node) - return unless node.filter_type == 'javascript' - - record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') - end - - def visit_tag(node) - return unless node.tag_name == 'script' - - record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') - end - end - end -end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index e2083848a8d..722e3e04d1c 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -122,7 +122,7 @@ module Mattermost @oauth_uri = nil - response = get('/oauth/gitlab/login', follow_redirects: false, format: 'text/html') + response = get('/oauth/gitlab/login', follow_redirects: false) return unless (300...400) === response.code redirect_uri = response.headers['location'] diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb new file mode 100644 index 00000000000..4c8cb6e97cc --- /dev/null +++ b/lib/quality/seeders/issues.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# rubocop:disable CodeReuse/ActiveRecord +module Quality + module Seeders + class Issues + DEFAULT_BACKFILL_WEEKS = 52 + DEFAULT_AVERAGE_ISSUES_PER_WEEK = 10 + + attr_reader :project, :user + + def initialize(project:) + @project = project + end + + def seed(backfill_weeks: DEFAULT_BACKFILL_WEEKS, average_issues_per_week: DEFAULT_AVERAGE_ISSUES_PER_WEEK) + created_at = backfill_weeks.to_i.weeks.ago + team = project.team.users + created_issues_count = 0 + + loop do + rand(average_issues_per_week * 2).times do + params = { + title: FFaker::Lorem.sentence(6), + description: FFaker::Lorem.sentence, + created_at: created_at + rand(6).days, + state: %w[opened closed].sample, + milestone: project.milestones.sample, + assignee_ids: Array(team.pluck(:id).sample(3)), + labels: labels.join(',') + } + issue = ::Issues::CreateService.new(project, team.sample, params).execute + + if issue.persisted? + created_issues_count += 1 + print '.' # rubocop:disable Rails/Output + end + end + + created_at += 1.week + + break if created_at > Time.now + end + + created_issues_count + end + + private + + def labels + @labels_pool ||= project.labels.limit(rand(3)).pluck(:title).tap do |labels_array| + labels_array.concat(project.group.labels.limit(rand(3)).pluck(:title)) if project.group + end + end + end + end +end +# rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb new file mode 100644 index 00000000000..24d8eac200c --- /dev/null +++ b/lib/quality/test_level.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Quality + class TestLevel + UnknownTestLevelError = Class.new(StandardError) + + TEST_LEVEL_FOLDERS = { + unit: %w[ + bin + config + db + dependencies + factories + finders + frontend + graphql + helpers + initializers + javascripts + lib + migrations + models + policies + presenters + rack_servers + routing + rubocop + serializers + services + sidekiq + tasks + uploaders + validators + views + workers + elastic_integration + ], + integration: %w[ + controllers + mailers + requests + ], + system: ['features'] + }.freeze + + attr_reader :prefix + + def initialize(prefix = nil) + @prefix = prefix + @patterns = {} + @regexps = {} + end + + def pattern(level) + @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb".freeze + end + + def regexp(level) + @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze + end + + def level_for(file_path) + case file_path + when regexp(:unit) + :unit + when regexp(:integration) + :integration + when regexp(:system) + :system + else + raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}." + end + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 2f2de083dc0..32df74f104a 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -1,8 +1,8 @@ #! /bin/sh # GITLAB -# Maintainer: @randx -# Authors: rovanion.luckey@gmail.com, @randx +# Maintainer: @dzaporozhets +# Authors: rovanion.luckey@gmail.com, @dzaporozhets ### BEGIN INIT INFO # Provides: gitlab @@ -26,6 +26,7 @@ ### Environment variables RAILS_ENV="production" +EXPERIMENTAL_PUMA="" # Script variable names should be lower-case not to conflict with # internal /bin/sh variables such as PATH, EDITOR or SHELL. @@ -75,7 +76,7 @@ check_pids(){ echo "Could not create the path $pid_path needed to store the pids." exit 1 fi - # If there exists a file which should hold the value of the Unicorn pid: read it. + # If there exists a file which should hold the value of the web server pid: read it. if [ -f "$web_server_pid_path" ]; then wpid=$(cat "$web_server_pid_path") else @@ -198,7 +199,7 @@ check_stale_pids(){ # If there is a pid it is something else than 0, the service is running if # *_status is == 0. if [ "$wpid" != "0" ] && [ "$web_status" != "0" ]; then - echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran." + echo "Removing stale web server pid. This is most likely caused by the web server crashing the last time it ran." if ! rm "$web_server_pid_path"; then echo "Unable to remove stale pid, exiting." exit 1 @@ -250,12 +251,12 @@ exit_if_not_running(){ fi } -## Starts Unicorn and Sidekiq if they're not running. +## Starts web server and Sidekiq if they're not running. start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab Unicorn" + echo "Starting GitLab web server" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -275,12 +276,12 @@ start_gitlab() { # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then - echo "The Unicorn web server already running with pid $wpid, not restarting." + echo "The web server already running with pid $wpid, not restarting." else # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV bin/web start + RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start fi # If sidekiq is already running, don't start it again. @@ -336,13 +337,13 @@ start_gitlab() { print_status } -## Asks Unicorn, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them. +## Asks web server, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them. stop_gitlab() { exit_if_not_running if [ "$web_status" = "0" ]; then - echo "Shutting down GitLab Unicorn" - RAILS_ENV=$RAILS_ENV bin/web stop + echo "Shutting down GitLab web server" + RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -398,9 +399,9 @@ print_status() { return fi if [ "$web_status" = "0" ]; then - echo "The GitLab Unicorn web server with pid $wpid is running." + echo "The GitLab web server with pid $wpid is running." else - printf "The GitLab Unicorn web server is \033[31mnot running\033[0m.\n" + printf "The GitLab web server is \033[31mnot running\033[0m.\n" fi if [ "$sidekiq_status" = "0" ]; then echo "The GitLab Sidekiq job dispatcher with pid $spid is running." @@ -438,15 +439,15 @@ print_status() { fi } -## Tells unicorn to reload its config and Sidekiq to restart +## Tells web server to reload its config and Sidekiq to restart reload_gitlab(){ exit_if_not_running if [ "$wpid" = "0" ];then - echo "The GitLab Unicorn Web server is not running thus its configuration can't be reloaded." + echo "The GitLab web server Web server is not running thus its configuration can't be reloaded." exit 1 fi - printf "Reloading GitLab Unicorn configuration... " - RAILS_ENV=$RAILS_ENV bin/web reload + printf "Reloading GitLab web server configuration... " + RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." @@ -461,7 +462,7 @@ reload_gitlab(){ print_status } -## Restarts Sidekiq and Unicorn. +## Restarts Sidekiq and web server. restart_gitlab(){ check_status if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 295c79fccfc..ab41dba3017 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -5,6 +5,9 @@ # Normal values are "production", "test" and "development". RAILS_ENV="production" +# Uncomment the line below to enable Puma web server instead of Unicorn. +# EXPERIMENTAL_PUMA=1 + # app_user defines the user that GitLab is run as. # The default is "git". app_user="git" diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 46aad8aa885..c36cacbaf4f 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -121,7 +121,7 @@ module SystemCheck # # @see #try_fixing_it # @see #fix_and_rerun - # @see #for_more_infromation + # @see #for_more_information def show_error raise NotImplementedError end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 560a52053d8..8cf7c9e89f0 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -209,7 +209,7 @@ namespace :gemojione do image.destroy! end - EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i + EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i.freeze def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) # Rename file from unicode to emoji name matches = EMOJI_IMAGE_PATH_RE.match(image_path) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3977fc7ad8c..c531eb1d216 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -20,6 +20,11 @@ namespace :gitlab do backup.pack backup.cleanup backup.remove_old + + puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + "and are not included in this backup. You will need these files to restore a backup.\n" \ + "Please back them up manually.".color(:red) + puts "Backup task is done." end # Restore backup of GitLab system @@ -68,6 +73,9 @@ namespace :gitlab do Rake::Task['cache:clear'].invoke backup.cleanup + puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + "and are not included in this backup. You will need to restore these files manually.".color(:red) + puts "Restore task is done." end namespace :repo do diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake index d115961108e..d88bcca0819 100644 --- a/lib/tasks/gitlab/features.rake +++ b/lib/tasks/gitlab/features.rake @@ -17,7 +17,7 @@ namespace :gitlab do if status Feature.enable(flag) else - Feature.disable(flag) + Feature.get(flag).remove end end end diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake new file mode 100644 index 00000000000..155ba979b36 --- /dev/null +++ b/lib/tasks/gitlab/seed.rake @@ -0,0 +1,34 @@ +namespace :gitlab do + namespace :seed do + desc "GitLab | Seed | Seeds issues" + task :issues, [:project_full_path] => :environment do |t, args| + projects = + if args.project_full_path + project = Project.find_by_full_path(args.project_full_path) + + unless project + error_message = "Project '#{args.project_full_path}' does not exist!" + potential_projects = Project.search(args.project_full_path) + + if potential_projects.present? + error_message += " Did you mean '#{potential_projects.first.full_path}'?" + end + + puts error_message.color(:red) + exit 1 + end + + [project] + else + Project.find_each + end + + projects.each do |project| + puts "\nSeeding issues for the '#{project.full_path}' project" + seeder = Quality::Seeders::Issues.new(project: project) + issues_created = seeder.seed(backfill_weeks: 5, average_issues_per_week: 2) + puts "\n#{issues_created} issues created!" + end + end + end +end diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index 786efd14b1a..305e15d69d5 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -1,6 +1,6 @@ unless Rails.env.production? require 'haml_lint/rake_task' - require 'haml_lint/inline_javascript' + require Rails.root.join('haml_lint/inline_javascript') # Workaround for warnings from parser/current # Keep it even if it no longer emits any warnings, diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index c5d0f2c292f..2353b2dc659 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -37,32 +37,15 @@ unless Rails.env.production? lint:static_verification ].each do |task| pid = Process.fork do - rd_out, wr_out = IO.pipe - rd_err, wr_err = IO.pipe - stdout = $stdout.dup - stderr = $stderr.dup - $stdout.reopen(wr_out) - $stderr.reopen(wr_err) - - begin - Rake::Task[task].invoke - rescue SystemExit => ex - msg = "*** Rake task #{task} exited:" - raise ex - rescue => ex - msg = "*** Rake task #{task} raised #{ex.class}:" - raise ex - ensure - $stdout.reopen(stdout) - $stderr.reopen(stderr) - wr_out.close - wr_err.close - - warn "\n#{msg}\n\n" if msg - - IO.copy_stream(rd_out, $stdout) - IO.copy_stream(rd_err, $stderr) - end + puts "*** Running rake task: #{task} ***" + + Rake::Task[task].invoke + rescue SystemExit => ex + warn "!!! Rake task #{task} exited:" + raise ex + rescue StandardError, ScriptError => ex + warn "!!! Rake task #{task} raised #{ex.class}:" + raise ex end Process.waitpid(pid) diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 2eddcb3c777..c881ad4cf12 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -1,7 +1,32 @@ +# frozen_string_literal: true + +return if Rails.env.production? + Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') namespace :spec do - desc 'GitLab | Rspec | Run request specs' + desc 'GitLab | RSpec | Run unit tests' + RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:unit) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run integration tests' + RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:integration) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run system tests' + RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:system) + t.rspec_opts = args[:rspec_opts] + end + + desc '[Deprecated] Use the "bin/rspec --tag api" instead' task :api do cmds = [ %w(rake gitlab:setup), @@ -10,7 +35,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run feature specs' + desc '[Deprecated] Use the "spec:system" task instead' task :feature do cmds = [ %w(rake gitlab:setup), @@ -19,7 +44,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run model specs' + desc '[Deprecated] Use "bin/rspec spec/models" instead' task :models do cmds = [ %w(rake gitlab:setup), @@ -28,7 +53,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run service specs' + desc '[Deprecated] Use "bin/rspec spec/services" instead' task :services do cmds = [ %w(rake gitlab:setup), @@ -37,7 +62,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run lib specs' + desc '[Deprecated] Use "bin/rspec spec/lib" instead' task :lib do cmds = [ %w(rake gitlab:setup), @@ -45,15 +70,6 @@ namespace :spec do ] run_commands(cmds) end - - desc 'GitLab | Rspec | Run other specs' - task :other do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) - ] - run_commands(cmds) - end end desc "GitLab | Run specs" |