diff options
Diffstat (limited to 'lib')
100 files changed, 1746 insertions, 581 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f4a96b9711b..20f8c637274 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -98,7 +98,6 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages - mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistry diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb deleted file mode 100644 index da756daadcc..00000000000 --- a/lib/api/circuit_breakers.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module API - class CircuitBreakers < Grape::API - before { authenticated_as_admin! } - - resource :circuit_breakers do - params do - requires :type, - type: String, - desc: "The type of circuitbreaker", - values: ['repository_storage'] - end - resource ':type' do - namespace '', requirements: { type: 'repository_storage' } do - desc 'Get all git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get do - present [] - end - - desc 'Get all failing git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get 'failing' do - present [] - end - - desc 'Reset all storage failures and open circuitbreaker' do - detail 'This feature was introduced in GitLab 9.5' - end - delete do - end - end - end - end - end -end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 65eb9bfb87e..80913f4ca07 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -96,17 +96,27 @@ module API end end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' end post ':id/repository/commits' do + if params[:start_project] + start_project = find_project!(params[:start_project]) + + unless user_project.forked_from?(start_project) + forbidden!("Project is not included in the fork network for #{start_project.full_name}") + end + end + authorize_push_to_branch!(params[:branch]) attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 5928ee1657b..693172b7d08 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -206,7 +206,7 @@ module API delete_note(noteable, params[:note_id]) end - if Noteable::RESOLVABLE_TYPES.include?(noteable_type.to_s) + if Noteable.resolvable_types.include?(noteable_type.to_s) desc "Resolve/unresolve an existing #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1a3318fe849..b1b6e7bd7b9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -239,6 +239,7 @@ module API end end + expose :empty_repo?, as: :empty_repo expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } @@ -302,6 +303,7 @@ module API expose :commit_count expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end @@ -354,6 +356,7 @@ module API with_options format_with: -> (value) { value.to_i } do expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end @@ -696,7 +699,7 @@ module API # See https://gitlab.com/gitlab-org/gitlab-ce/issues/42344 for more # information. expose :merge_status do |merge_request| - merge_request.check_if_can_be_merged + merge_request.check_mergeability merge_request.merge_status end expose :diff_head_sha, as: :sha @@ -1303,6 +1306,7 @@ module API class Variable < Grape::Entity expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } + expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } end class Pipeline < PipelineBasic diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 6893c8c40be..ec1020c7c78 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,34 +7,9 @@ module API before { authenticate_non_get! } - helpers do - params :optional_params_ce do - optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, - values: Gitlab::VisibilityLevel.string_values, - default: Gitlab::VisibilityLevel.string_level( - Gitlab::CurrentSettings.current_application_settings.default_group_visibility), - desc: 'The visibility of the group' - optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' - end - - params :optional_params_ee do - end - - params :optional_update_params_ee do - end - end - - include ::API::Helpers::GroupsHelpers + helpers Helpers::GroupsHelpers helpers do - params :optional_params do - use :optional_params_ce - use :optional_params_ee - end - params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 7e4539d0419..00bcf6b055b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -445,7 +445,7 @@ module API end def present_carrierwave_file!(file, supports_direct_download: true) - return not_found! unless file.exists? + return not_found! unless file&.exists? if file.file_storage? present_disk_file!(file.path, file.filename) diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index ae677547760..2c33d79f6c8 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -4,6 +4,30 @@ module API module Helpers module GroupsHelpers extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ce do + optional :description, type: String, desc: 'The description of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' + end + + params :optional_params_ee do + end + + params :optional_update_params_ee do + end + + params :optional_params do + use :optional_params_ce + use :optional_params_ee + end end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 71c30ec99a5..c318f5b9127 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -46,6 +46,8 @@ module API def process_mr_push_options(push_options, project, user, changes) output = {} + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/61359') + service = ::MergeRequests::PushOptionsHandlerService.new( project, user, diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index fc66cec5341..5b7199fddb0 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -3,6 +3,14 @@ module API module Helpers module IssuesHelpers + extend Grape::API::Helpers + + params :optional_issue_params_ee do + end + + params :optional_issues_params_ee do + end + def self.update_params_at_least_one_of [ :assignee_id, diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 73d58ee7f37..1395ffadab9 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -19,28 +19,13 @@ module API .non_request end - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_project(project) - shared_group_ids = project.project_group_links.pluck(:group_id) - project_group_ids = project.group&.self_and_ancestors&.pluck(:id) - source_ids = [project.id, project_group_ids, shared_group_ids] - .flatten - .compact - Member.includes(:user) - .joins(user: :project_authorizations) - .where(project_authorizations: { project_id: project.id }) - .where(source_id: source_ids) + MembersFinder.new(project, current_user).execute(include_invited_groups_members: true) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_group(group) - source_ids = group.self_and_ancestors.pluck(:id) - Member.includes(:user) - .where(source_id: source_ids) - .where(source_type: 'Namespace') + GroupMembersFinder.new(group).execute end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index aaf32dafca4..813e46e9520 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -4,48 +4,45 @@ module API module Helpers module ProjectsHelpers extend ActiveSupport::Concern + extend Grape::API::Helpers - included do - helpers do - params :optional_project_params_ce do - optional :description, type: String, desc: 'The description of the project' - optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' - optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' - optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' - optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' - optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' - optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' - optional :public_builds, type: Boolean, desc: 'Perform public builds' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - optional :tag_list, type: Array[String], desc: 'The list of tags for a project' - optional :avatar, type: File, desc: 'Avatar image for project' - optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' - optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' - optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" - optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' - end + params :optional_project_params_ce do + optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' + optional :avatar, type: File, desc: 'Avatar image for project' + optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' + optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" + optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' + end + + params :optional_project_params_ee do + end - if Gitlab.ee? - params :optional_project_params_ee do - optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' - optional :approvals_before_merge, type: Integer, desc: 'How many approvers should approve merge request by default' - optional :mirror, type: Boolean, desc: 'Enables pull mirroring in a project' - optional :mirror_trigger_builds, type: Boolean, desc: 'Pull mirroring triggers builds' - end - end + params :optional_project_params do + use :optional_project_params_ce + use :optional_project_params_ee + end + + params :optional_filter_params_ee do + end - params :optional_project_params do - use :optional_project_params_ce - use :optional_project_params_ee if Gitlab.ee? - end - end + params :optional_update_params_ee do end def self.update_params_at_least_one_of diff --git a/lib/api/helpers/protected_branches_helpers.rb b/lib/api/helpers/protected_branches_helpers.rb new file mode 100644 index 00000000000..0fc6841d79a --- /dev/null +++ b/lib/api/helpers/protected_branches_helpers.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Helpers + module ProtectedBranchesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + end + end +end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 953be7f3798..44c577204b8 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -563,6 +563,12 @@ module API name: :notify_only_broken_pipelines, type: Boolean, desc: 'Notify only broken pipelines' + }, + { + required: false, + name: :notify_only_default_branch, + type: Boolean, + desc: 'Send notifications only for the default branch' } ], 'pivotaltracker' => [ diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb new file mode 100644 index 00000000000..6441bb579ff --- /dev/null +++ b/lib/api/helpers/settings_helpers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Helpers + module SettingsHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + + def self.optional_attributes + [*::ApplicationSettingsHelper.visible_attributes, + *::ApplicationSettingsHelper.external_authorization_service_attributes, + :performance_bar_allowed_group_id].freeze + end + end + end +end diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb new file mode 100644 index 00000000000..56fd3c6602d --- /dev/null +++ b/lib/api/helpers/users_helpers.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Helpers + module UsersHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + + params :optional_index_params_ee do + end + end + end +end diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb new file mode 100644 index 00000000000..78a92d0f5a6 --- /dev/null +++ b/lib/api/helpers/variables_helpers.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Helpers + module VariablesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index c82fd230d7a..224aaaaf006 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -264,12 +264,8 @@ module API PostReceive.perform_async(params[:gl_repository], params[:identifier], params[:changes], push_options.as_json) - if Feature.enabled?(:mr_push_options, default_enabled: true) - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/61359') - - mr_options = push_options.get(:merge_request) - output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? - end + mr_options = push_options.get(:merge_request) + output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0b4da01f3c8..56960a2eb64 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -9,16 +9,6 @@ module API before { authenticate_non_get! } helpers do - if Gitlab.ee? - params :issues_params_ee do - optional :weight, types: [Integer, String], integer_none_any: true, desc: 'The weight of the issue' - end - - params :issue_params_ee do - optional :weight, type: Integer, desc: 'The weight of the issue' - end - end - params :issues_stats_params do optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' @@ -47,7 +37,7 @@ module API optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' - use :issues_params_ee if Gitlab.ee? + use :optional_issues_params_ee end params :issues_params do @@ -73,7 +63,7 @@ module API optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - use :issue_params_ee if Gitlab.ee? + use :optional_issue_params_ee end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index ce85772e4ed..5bbf6df78b0 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -386,9 +386,8 @@ module API ) if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) + AutoMergeService.new(merge_request.target_project, current_user, merge_params) + .execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) else ::MergeRequests::MergeService .new(merge_request.target_project, current_user, merge_params) @@ -398,28 +397,16 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - desc 'Merge a merge request to its default temporary merge ref path' - params do - optional :merge_commit_message, type: String, desc: 'Custom merge commit message' - end - put ':id/merge_requests/:merge_request_iid/merge_to_ref' do + desc 'Returns the up to date merge-ref HEAD commit' + get ':id/merge_requests/:merge_request_iid/merge_ref' do merge_request = find_project_merge_request(params[:merge_request_iid]) - authorize! :admin_merge_request, user_project - - merge_params = { - commit_message: params[:merge_commit_message] - } - - result = ::MergeRequests::MergeToRefService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) + result = ::MergeRequests::MergeabilityCheckService.new(merge_request).execute - if result[:status] == :success - present result.slice(:commit_id), 200 + if result.success? + present :commit_id, result.payload.dig(:merge_ref_head, :commit_id) else - http_status = result[:http_status] || 400 - render_api_error!(result[:message], http_status) + render_api_error!(result.message, 400) end end @@ -429,11 +416,9 @@ module API post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do merge_request = find_project_merge_request(params[:merge_request_iid]) - unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) + unauthorized! unless merge_request.can_cancel_auto_merge?(current_user) - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user) - .cancel(merge_request) + AutoMergeService.new(merge_request.target_project, current_user).cancel(merge_request) end desc 'Rebase the merge request against its target branch' do diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index c64ec2fcc95..71891e43dcc 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -3,7 +3,8 @@ module API class ProjectImport < Grape::API include PaginationParams - include Helpers::ProjectsHelpers + + helpers Helpers::ProjectsHelpers helpers do def import_params diff --git a/lib/api/projects.rb b/lib/api/projects.rb index cb0106592f5..1e14c77b5be 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -6,27 +6,12 @@ module API class Projects < Grape::API include PaginationParams include Helpers::CustomAttributes - include Helpers::ProjectsHelpers + + helpers Helpers::ProjectsHelpers before { authenticate_non_get! } helpers do - if Gitlab.ee? - params :optional_filter_params_ee do - optional :wiki_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where wiki checksum is failed' - optional :repository_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where repository checksum is failed' - end - - params :optional_update_params_ee do - optional :mirror_user_id, type: Integer, desc: 'User responsible for all the activity surrounding a pull mirror event' - optional :only_mirror_protected_branches, type: Grape::API::Boolean, desc: 'Only mirror protected branches' - optional :mirror_overwrites_diverged_branches, type: Grape::API::Boolean, desc: 'Pull mirror overwrites diverged branches' - optional :import_url, type: String, desc: 'URL from which the project is imported' - optional :packages_enabled, type: Grape::API::Boolean, desc: 'Enable project packages feature' - optional :fallback_approvals_required, type: Integer, desc: 'Overall approvals required when no rule is present' - end - end - # EE::API::Projects would override this method def apply_filters(projects) projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] @@ -77,7 +62,7 @@ module API optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' - use :optional_filter_params_ee if Gitlab.ee? + use :optional_filter_params_ee end params :create_params do @@ -296,7 +281,7 @@ module API optional :path, type: String, desc: 'The path of the repository' use :optional_project_params - use :optional_update_params_ee if Gitlab.ee? + use :optional_update_params_ee at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index f8cce1ed784..33dea25289a 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -8,6 +8,8 @@ module API before { authorize_admin_project } + helpers Helpers::ProtectedBranchesHelpers + params do requires :id, type: String, desc: 'The ID of a project' end @@ -52,29 +54,7 @@ module API values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' - if Gitlab.ee? - optional :unprotect_access_level, type: Integer, - values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels, - desc: 'Access levels allowed to unprotect (defaults: `40`, maintainer access level)' - - optional :allowed_to_push, type: Array, desc: 'An array of users/groups allowed to push' do - optional :access_level, type: Integer, values: ProtectedBranch::PushAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - - optional :allowed_to_merge, type: Array, desc: 'An array of users/groups allowed to merge' do - optional :access_level, type: Integer, values: ProtectedBranch::MergeAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - - optional :allowed_to_unprotect, type: Array, desc: 'An array of users/groups allowed to unprotect' do - optional :access_level, type: Integer, values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - end + use :optional_params_ee end # rubocop: disable CodeReuse/ActiveRecord post ':id/protected_branches' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 8046acfa397..6767ef882cb 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -4,6 +4,8 @@ module API class Settings < Grape::API before { authenticated_as_admin! } + helpers Helpers::SettingsHelpers + helpers do def current_settings @current_setting ||= @@ -136,54 +138,10 @@ module API desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." end - if Gitlab.ee? - optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch' - - given elasticsearch_aws: ->(val) { val } do - optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key' - requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured' - optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key' - end - - optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing' - - given elasticsearch_indexing: ->(val) { val } do - optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search' - requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")' - optional :elasticsearch_limit_indexing, type: Boolean, desc: 'Limit Elasticsearch to index certain namespaces and projects' - end - - given elasticsearch_limit_indexing: ->(val) { val } do - optional :elasticsearch_namespace_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The namespace ids to index with Elasticsearch.' - optional :elasticsearch_project_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The project ids to index with Elasticsearch.' - end - - optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons' - optional :help_text, type: String, desc: 'GitLab server administrator information' - optional :repository_size_limit, type: Integer, desc: 'Size limit per repository (MB)' - optional :file_template_project_id, type: Integer, desc: 'ID of project where instance-level file templates are stored.' - optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.' - optional :snowplow_enabled, type: Boolean, desc: 'Enable Snowplow' - - given snowplow_enabled: ->(val) { val } do - requires :snowplow_collector_uri, type: String, desc: 'Snowplow Collector URI' - optional :snowplow_cookie_domain, type: String, desc: 'Snowplow cookie domain' - optional :snowplow_site_id, type: String, desc: 'Snowplow Site/Application ID' - end - - optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' - end - - optional_attributes = [*::ApplicationSettingsHelper.visible_attributes, - *::ApplicationSettingsHelper.external_authorization_service_attributes, - :performance_bar_allowed_group_id] - - if Gitlab.ee? - optional_attributes += EE::ApplicationSettingsHelper.possible_licensed_attributes - end + use :optional_params_ee - optional(*optional_attributes) - at_least_one_of(*optional_attributes) + optional(*Helpers::SettingsHelpers.optional_attributes) + at_least_one_of(*Helpers::SettingsHelpers.optional_attributes) end put "application/settings" do attrs = declared_params(include_missing: false) diff --git a/lib/api/users.rb b/lib/api/users.rb index 2f23e33bd4a..6afeebb6890 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -15,6 +15,8 @@ module API authenticate_non_get! end + helpers Helpers::UsersHelpers + helpers do # rubocop: disable CodeReuse/ActiveRecord def find_user_by_id(params) @@ -52,10 +54,7 @@ module API optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider - if Gitlab.ee? - optional :shared_runners_minutes_limit, type: Integer, desc: 'Pipeline minutes quota for this user' - optional :extra_shared_runners_minutes_limit, type: Integer, desc: '(admin-only) Extra pipeline minutes quota for this user' - end + use :optional_params_ee end params :sort_params do @@ -85,10 +84,7 @@ module API use :sort_params use :pagination use :with_custom_attributes - - if Gitlab.ee? - optional :skip_ldap, type: Boolean, default: false, desc: 'Skip LDAP users' - end + use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord get do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index a1bb21b3a06..af1d7936556 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,6 +7,8 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + helpers Helpers::VariablesHelpers + helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain @@ -54,12 +56,11 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - if Gitlab.ee? - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - end + use :optional_params_ee end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -80,12 +81,11 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - if Gitlab.ee? - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - end + use :optional_params_ee end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 44b151d01e7..0224dd8fcd1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -363,6 +363,14 @@ module Banzai group_ref end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 4d67140b0a1..4892668fc22 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -104,14 +104,6 @@ module Banzai matches[:namespace] && matches[:project] end - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end - - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end - def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index fce042e8946..08969753d75 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -51,13 +51,13 @@ module Banzai # default implementation. return super(text, pattern) if pattern != Milestone.reference_pattern - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone yield match, milestone.id, $~[:project], $~[:namespace], $~ else - match + escape_html_entities(match) end end end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index f4cc8beeb52..77b5053f38c 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -4,6 +4,8 @@ module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter class Rewriter + UNSAFE_SLUG_REGEXES = [/\Ajavascript:/i].freeze + def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) @wiki_base_path = wiki && wiki.wiki_base_path @@ -35,6 +37,8 @@ module Banzai # Of the form `./link`, `../link`, or similar def apply_hierarchical_link_rules! + return if slug_considered_unsafe? + @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' end @@ -54,6 +58,10 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end + + def slug_considered_unsafe? + UNSAFE_SLUG_REGEXES.any? { |r| r.match?(@slug) } + end end end end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 7db5f5e1f7d..c2da7fec7cc 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -70,8 +70,11 @@ module Banzai # Build the raw <a> tag just with a link as href and content if # it's originally a link pattern. We shouldn't return a plain text href. original_link = - if link_reference == 'true' && href = original_content - %(<a href="#{href}">#{href}</a>) + if link_reference == 'true' + href = node.attr('href') + content = original_content + + %(<a href="#{href}">#{content}</a>) end # The reference should be replaced by the original link's content, diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 3f107fbbf3b..ccaf06c5d6a 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -40,6 +40,7 @@ module Gitlab SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze + HTTP_PROXY_ENV_VARS = %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY).freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com @@ -66,6 +67,10 @@ module Gitlab end end + def self.http_proxy_env? + HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] } + end + def self.process_name return 'sidekiq' if Sidekiq.server? return 'console' if defined?(Rails::Console) diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 36fc8061d92..72a187377d0 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -55,7 +55,7 @@ module Gitlab private def info - auth_hash.info + auth_hash['info'] end def get_info(key) diff --git a/lib/gitlab/auth_logger.rb b/lib/gitlab/auth_logger.rb new file mode 100644 index 00000000000..6d3edba02b0 --- /dev/null +++ b/lib/gitlab/auth_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class AuthLogger < Gitlab::JsonLogger + def self.file_name_noext + 'auth' + end + end +end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb new file mode 100644 index 00000000000..886c41a2b9d --- /dev/null +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CalculateWikiSizes + def perform(start_id, stop_id) + ::ProjectStatistics.where(wiki_size: nil) + .where(id: start_id..stop_id) + .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| + statistics.refresh!(only: [:wiki_size]) + rescue => e + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + end + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb new file mode 100644 index 00000000000..447fec8903c --- /dev/null +++ b/lib/gitlab/background_migration/reset_merge_status.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Updates the range of given MRs to merge_status "unchecked", if they're opened + # and mergeable. + class ResetMergeStatus + def perform(from_id, to_id) + relation = MergeRequest.where(id: from_id..to_id, + state: 'opened', + merge_status: 'can_be_merged') + + relation.update_all(merge_status: 'unchecked') + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 0beeb44c272..21c42857895 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -24,7 +24,7 @@ module Gitlab end entry :ports, Entry::Ports, - description: 'Ports used expose the image' + description: 'Ports used to expose the image' attributes :ports diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 084fa4047a4..8d16371e857 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -24,6 +24,9 @@ module Gitlab validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } end + entry :ports, Entry::Ports, + description: 'Ports used to expose the service' + def alias value[:alias] end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 939112e6e41..32ab216dd55 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,8 +50,8 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.9 - HELM_VERSION: 2.13.1 + KUBERNETES_VERSION: 1.11.10 + HELM_VERSION: 2.14.0 DOCKER_DRIVER: overlay2 @@ -89,4 +89,4 @@ include: dast: except: refs: - - master
\ No newline at end of file + - master diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 779f4b5f006..1d55c64ec56 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -432,6 +432,7 @@ rollout 100%: --set postgresql.postgresUser="$POSTGRES_USER" \ --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set postgresql.imageTag="$POSTGRES_VERSION" \ --set application.migrateCommand="$DB_MIGRATE" \ $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ @@ -507,23 +508,13 @@ rollout 100%: kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } - # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN - function ensure_kube_ingress_base_domain() { - if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then - export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN - fi - } - function check_kube_domain() { - ensure_kube_ingress_base_domain - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then echo "In order to deploy or use Review Apps," - echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" echo "or by defining a variable at group or project level." echo "You can also manually add it in .gitlab-ci.yml" - echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" false else true diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index c9838c7a7ff..08dc74e041a 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,18 +1,14 @@ ---- +# This file is a template, and might need editing before it works on your project. + # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # # For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html -# -# This template will build and test your projects as well as create the documentation. -# + +# This template will build and test your projects # * Caches downloaded dependencies and plugins between invocation. # * Verify but don't deploy merge requests. # * Deploy built artifacts from master branch only. -# * Shows how to use multiple jobs in test stage for verifying functionality -# with multiple JDKs. -# * Uses site:stage to collect the documentation for multi-module projects. -# * Publishes the documentation for `master` branch. variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. @@ -23,78 +19,38 @@ variables: # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" +# This template uses jdk8 for verifying and deploying images +image: maven:3.3.9-jdk-8 + # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' cache: paths: - .m2/repository -# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. -# Because some enforcer rules might check dependency convergence and class duplications -# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. -.validate: &validate - stage: build - script: - - 'mvn $MAVEN_CLI_OPTS test-compile' - # For merge requests do not `deploy` but only run `verify`. # See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html .verify: &verify stage: test script: - - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + - 'mvn $MAVEN_CLI_OPTS verify' except: - master -# Validate merge requests using JDK7 -validate:jdk7: - <<: *validate - image: maven:3.3.9-jdk-7 - -# Validate merge requests using JDK8 -validate:jdk8: - <<: *validate - image: maven:3.3.9-jdk-8 - -# Verify merge requests using JDK7 -verify:jdk7: - <<: *verify - image: maven:3.3.9-jdk-7 - # Verify merge requests using JDK8 verify:jdk8: <<: *verify - image: maven:3.3.9-jdk-8 +# To deploy packages from CI, create a ci_settings.xml file +# For deploying packages to GitLab's Maven Repository: See https://gitlab.com/help/user/project/packages/maven_repository.md#creating-maven-packages-with-gitlab-cicd for more details. +# Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. # For `master` branch run `mvn deploy` automatically. -# Here you need to decide whether you want to use JDK7 or 8. -# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. -# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. -# See https://maven.apache.org/settings.html deploy:jdk8: - # Use stage test here, so the pages job may later pickup the created site. - stage: test - script: - - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' - only: - - master - # Archive up the built documentation site. - artifacts: - paths: - - target/staging - image: maven:3.3.9-jdk-8 - -pages: - image: busybox:latest stage: deploy script: - # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, - # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. - - mv target/staging public - dependencies: - - deploy:jdk8 - artifacts: - paths: - - public + - if [ ! -f ci_settings.xml ]; + then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://gitlab.com/help/user/project/packages/maven_repository.md#creating-maven-packages-with-gitlab-cicd for instructions."; + fi + - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: - master diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 324e39c7747..5372ec6cceb 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -23,6 +23,9 @@ container_scanning: DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 + ## Disable the proxy for clair-local-scan, otherwise Container Scanning will + ## fail when a proxy is used. + NO_PROXY: ${DOCKER_SERVICE},localhost allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 706692e063b..abf16e5b2e7 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -40,6 +40,7 @@ sast: SAST_BRAKEMAN_LEVEL \ SAST_GOSEC_LEVEL \ SAST_FLAWFINDER_LEVEL \ + SAST_GITLEAKS_ENTROPY_LEVEL \ SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b05dca409d1..e0f9eb59924 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -44,6 +44,14 @@ module Gitlab (@master_restart_hooks ||= []) << block end + def on_master_start(&block) + if in_clustered_environment? + on_before_fork(&block) + else + on_worker_start(&block) + end + end + # # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index f0ca397609d..7effb802678 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -1,6 +1,4 @@ # frozen_string_literal: true -require 'net/http' -require 'json' require_relative 'teammate' @@ -8,7 +6,6 @@ module Gitlab module Danger module Helper RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - ROULETTE_DATA_URL = URI.parse('https://about.gitlab.com/roulette.json').freeze # Returns a list of all files that have been added, modified or renamed. # `git.modified_files` might contain paths that already have been renamed, @@ -49,32 +46,6 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end - # Looks up the current list of GitLab team members and parses it into a - # useful form - # - # @return [Array<Teammate>] - def team - @team ||= - begin - rsp = Net::HTTP.get_response(ROULETTE_DATA_URL) - raise "Failed to read #{ROULETTE_DATA_URL}: #{rsp.code} #{rsp.message}" unless - rsp.is_a?(Net::HTTPSuccess) - - data = JSON.parse(rsp.body) - data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } - rescue JSON::ParserError - raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" - end - end - - # Like +team+, but only returns teammates in the current project, based on - # project_name. - # - # @return [Array<Teammate>] - def project_team - team.select { |member| member.in_project?(project_name) } - end - # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb new file mode 100644 index 00000000000..062eda38ee4 --- /dev/null +++ b/lib/gitlab/danger/roulette.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' +require 'cgi' + +require_relative 'teammate' + +module Gitlab + module Danger + module Roulette + ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json' + HTTPError = Class.new(RuntimeError) + + # Looks up the current list of GitLab team members and parses it into a + # useful form + # + # @return [Array<Teammate>] + def team + @team ||= + begin + data = http_get_json(ROULETTE_DATA_URL) + data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } + rescue JSON::ParserError + raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" + end + end + + # Like +team+, but only returns teammates in the current project, based on + # project_name. + # + # @return [Array<Teammate>] + def project_team(project_name) + team.select { |member| member.in_project?(project_name) } + end + + def canonical_branch_name(branch_name) + branch_name.gsub(/^[ce]e-|-[ce]e$/, '') + end + + def new_random(seed) + Random.new(Digest::MD5.hexdigest(seed).to_i(16)) + end + + # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the + # selection will change on next spin + def spin_for_person(people, random:) + person = nil + people = people.dup + + people.size.times do + person = people.sample(random: random) + + break person unless out_of_office?(person) + + people -= [person] + end + + person + end + + private + + def out_of_office?(person) + username = CGI.escape(person.username) + api_endpoint = "https://gitlab.com/api/v4/users/#{username}/status" + response = http_get_json(api_endpoint) + response["message"]&.match?(/OOO/i) + rescue HTTPError, JSON::ParserError + false # this is no worse than not checking for OOO + end + + def http_get_json(url) + rsp = Net::HTTP.get_response(URI.parse(url)) + + unless rsp.is_a?(Net::HTTPSuccess) + raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" + end + + JSON.parse(rsp.body) + end + end + end +end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index bfada512727..c4e66da8ed1 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -6,8 +6,8 @@ module Gitlab attr_reader :name, :username, :projects def initialize(options = {}) - @name = options['name'] @username = options['username'] + @name = options['name'] || @username @projects = options['projects'] end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index fa06fb935f7..e1e813849bf 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -19,7 +19,7 @@ module Gitlab def hook_attrs(pipeline) { id: pipeline.id, - ref: pipeline.ref, + ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, before_sha: pipeline.before_sha, diff --git a/lib/gitlab/diff/suggestion.rb b/lib/gitlab/diff/suggestion.rb index 027c7a31bcf..4a3ac2106e2 100644 --- a/lib/gitlab/diff/suggestion.rb +++ b/lib/gitlab/diff/suggestion.rb @@ -33,6 +33,8 @@ module Gitlab end def to_content + return "" if @text.blank? + # The parsed suggestion doesn't have information about the correct # ending characters (we may have a line break, or not), so we take # this information from the last line being changed (last diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 455588f3c66..dcdd3581d92 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -34,34 +34,6 @@ module Gitlab TagExistsError = Class.new(StandardError) ChecksumError = Class.new(StandardError) - class << self - def create_hooks(repo_path, global_hooks_path) - local_hooks_path = File.join(repo_path, 'hooks') - real_local_hooks_path = :not_found - - begin - real_local_hooks_path = File.realpath(local_hooks_path) - rescue Errno::ENOENT - # real_local_hooks_path == :not_found - end - - # Do nothing if hooks already exist - unless real_local_hooks_path == File.realpath(global_hooks_path) - if File.exist?(local_hooks_path) - # Move the existing hooks somewhere safe - FileUtils.mv( - local_hooks_path, - "#{local_hooks_path}.old.#{Time.now.to_i}") - end - - # Create the hooks symlink - FileUtils.ln_sf(global_hooks_path, local_hooks_path) - end - - true - end - end - # Directory name of repo attr_reader :name diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 3f13ebeb9d0..dfff6823689 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -5,12 +5,15 @@ module Gitlab module GitRefValidator extend self + + EXPANDED_PREFIXES = %w[refs/heads/ refs/remotes/].freeze + DISALLOWED_PREFIXES = %w[-].freeze + # Validates a given name against the git reference specification # # Returns true for a valid reference name, false otherwise def validate(ref_name) - not_allowed_prefixes = %w(refs/heads/ refs/remotes/ -) - return false if ref_name.start_with?(*not_allowed_prefixes) + return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) return false if ref_name == 'HEAD' begin @@ -19,5 +22,21 @@ module Gitlab return false end end + + def validate_merge_request_branch(ref_name) + return false if ref_name.start_with?(*DISALLOWED_PREFIXES) + + expanded_name = if ref_name.start_with?(*EXPANDED_PREFIXES) + ref_name + else + "refs/heads/#{ref_name}" + end + + begin + Rugged::Reference.valid_name?(expanded_name) + rescue ArgumentError + return false + end + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 05e06eec012..e683d4e5bbe 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,7 +33,7 @@ module Gitlab SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze # Server feature flags should use '_' to separate words. - SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE, 'delta_islands'].freeze + SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE].freeze MUTEX = Mutex.new diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 9d81441d96e..1d3541b80c7 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -29,29 +29,13 @@ module Gitlab end def execute - jid = generate_jid - - # The original import JID is the JID of the RepositoryImportWorker job, - # which will be removed once that job completes. Reusing that JID could - # result in StuckImportJobsWorker marking the job as stuck before we get - # to running Stage::ImportRepositoryWorker. - # - # We work around this by setting the JID to a custom generated one, then - # refreshing it in the various stages whenever necessary. - Gitlab::SidekiqStatus - .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - - project.import_state.update_column(:jid, jid) + Gitlab::Import::SetAsyncJid.set_jid(project) Stage::ImportRepositoryWorker .perform_async(project.id) true end - - def generate_jid - "github-importer/#{project.id}" - end end end end diff --git a/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb b/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb new file mode 100644 index 00000000000..5e151f4dbd7 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchProjectStatisticsLoader + attr_reader :project_id + + def initialize(project_id) + @project_id = project_id + end + + def find + BatchLoader.for(project_id).batch do |project_ids, loader| + ProjectStatistics.for_project_ids(project_ids).each do |statistics| + loader.call(statistics.project_id, statistics) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb b/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb deleted file mode 100644 index 95db130dbfc..00000000000 --- a/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module QueryAnalyzers - class LogQueryComplexity - class << self - def analyzer - GraphQL::Analysis::QueryComplexity.new do |query, complexity| - # temporary until https://gitlab.com/gitlab-org/gitlab-ce/issues/59587 - Rails.logger.info("[GraphQL Query Complexity] #{complexity} | admin? #{query.context[:current_user]&.admin?}") - end - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb new file mode 100644 index 00000000000..01b55a1667f --- /dev/null +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module QueryAnalyzers + class LoggerAnalyzer + COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } + DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } + + def analyze?(query) + Feature.enabled?(:graphql_logging, default_enabled: true) + end + + def initial_value(query) + variables = process_variables(query.provided_variables) + default_initial_values(query).merge({ + query_string: query.query_string, + variables: variables + }) + rescue => e + Gitlab::Sentry.track_exception(e) + default_initial_values(query) + end + + def call(memo, visit_type, irep_node) + memo + end + + def final_value(memo) + return if memo.nil? + + analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER] + complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers) + + memo[:depth] = depth + memo[:complexity] = complexity + memo[:duration] = duration(memo[:time_started]).round(1) + + GraphqlLogger.info(memo.except!(:time_started, :query)) + rescue => e + Gitlab::Sentry.track_exception(e) + end + + private + + def process_variables(variables) + if variables.respond_to?(:to_s) + variables.to_s + else + variables + end + end + + def duration(time_started) + nanoseconds = Gitlab::Metrics::System.monotonic_time - time_started + nanoseconds * 1000000 + end + + def default_initial_values(query) + { + time_started: Gitlab::Metrics::System.monotonic_time, + query_string: nil, + query: query, + variables: nil, + duration: nil + } + end + end + end + end +end diff --git a/lib/gitlab/graphql_logger.rb b/lib/gitlab/graphql_logger.rb new file mode 100644 index 00000000000..43d917908b6 --- /dev/null +++ b/lib/gitlab/graphql_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class GraphqlLogger < Gitlab::JsonLogger + def self.file_name_noext + 'graphql_json' + end + end +end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index bcd9e2be35f..db2b4dde244 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -9,9 +9,16 @@ module Gitlab BlockedUrlError = Class.new(StandardError) RedirectionTooDeep = Class.new(StandardError) + HTTP_ERRORS = [ + SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, + Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, + Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, + Gitlab::HTTP::RedirectionTooDeep + ].freeze + include HTTParty # rubocop:disable Gitlab/HTTParty - connection_adapter ProxyHTTPConnectionAdapter + connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) super diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index a64cb47e77e..41eab3658bc 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -10,17 +10,19 @@ # # This option will take precedence over the global setting. module Gitlab - class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter + class HTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - unless allow_local_requests? - begin - Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) - rescue Gitlab::UrlBlocker::BlockedUrlError => e - raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" - end + begin + @uri, hostname = Gitlab::UrlBlocker.validate!(uri, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection?) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" end - super + super.tap do |http| + http.hostname_override = hostname if hostname + end end private @@ -29,6 +31,12 @@ module Gitlab options.fetch(:allow_local_requests, allow_settings_local_requests?) end + def dns_rebind_protection? + return false if Gitlab.http_proxy_env? + + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end + def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? end diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb new file mode 100644 index 00000000000..584d24045ee --- /dev/null +++ b/lib/gitlab/import/set_async_jid.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# The original import JID is the JID of the RepositoryImportWorker job, +# which will be removed once that job completes. Reusing that JID could +# result in StuckImportJobsWorker marking the job as stuck before we get +# to running Stage::ImportRepositoryWorker. +# +# We work around this by setting the JID to a custom generated one, then +# refreshing it in the various stages whenever necessary. +module Gitlab + module Import + module SetAsyncJid + def self.set_jid(project) + jid = generate_jid(project) + + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.import_state.update_column(:jid, jid) + end + + def self.generate_jid(project) + "async-import/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 93b37b7bc5f..c28a1674018 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class AttributeCleaner ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_html\Z/).freeze def self.clean(*args) new(*args).clean @@ -24,7 +25,11 @@ module Gitlab private def prohibited_key?(key) - key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + key =~ PROHIBITED_REFERENCES && !permitted_key?(key) + end + + def permitted_key?(key) + ALLOWED_REFERENCES.include?(key) end def excluded_key?(key) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6be95a16513..a154de5419e 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -59,7 +59,11 @@ module Gitlab end def member_hash(member) - parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + parsed_hash(member).merge( + 'source_id' => @project.id, + 'importing' => true, + 'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min + ).except('user_id') end def parsed_hash(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 51001750a6c..20caadb89c0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -129,7 +129,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level - level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level + level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level { 'visibility_level' => level } end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 67952ca0f2d..e4d625b5738 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), - ImportSource.new('manifest', 'Manifest file', nil) + ImportSource.new('manifest', 'Manifest file', nil), + ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer) ].freeze class << self diff --git a/lib/gitlab/kubernetes/errors.rb b/lib/gitlab/kubernetes/errors.rb new file mode 100644 index 00000000000..81bf636eef7 --- /dev/null +++ b/lib/gitlab/kubernetes/errors.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Errors + CONNECTION = [ + SocketError, + OpenSSL::SSL::SSLError, + Errno::ECONNRESET, + Errno::ENETUNREACH, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Net::OpenTimeout, + Net::ReadTimeout, + IPAddr::InvalidAddressError + ].freeze + + AUTHENTICATION = [ + OpenSSL::X509::CertificateError + ].freeze + end + end +end diff --git a/lib/gitlab/lets_encrypt/client.rb b/lib/gitlab/lets_encrypt/client.rb index d7468b06767..66aea137012 100644 --- a/lib/gitlab/lets_encrypt/client.rb +++ b/lib/gitlab/lets_encrypt/client.rb @@ -3,6 +3,8 @@ module Gitlab module LetsEncrypt class Client + include Gitlab::Utils::StrongMemoize + PRODUCTION_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory' STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' @@ -35,6 +37,8 @@ module Gitlab def enabled? return false unless Feature.enabled?(:pages_auto_ssl) + return false unless private_key + Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted end @@ -45,7 +49,11 @@ module Gitlab end def private_key - @private_key ||= OpenSSL::PKey.read(Gitlab::Application.secrets.lets_encrypt_private_key) + strong_memoize(:private_key) do + private_key_string = Gitlab::CurrentSettings.lets_encrypt_private_key + private_key_string ||= generate_private_key + OpenSSL::PKey.read(private_key_string) if private_key_string + end end def admin_email @@ -69,6 +77,19 @@ module Gitlab STAGING_DIRECTORY_URL end end + + def generate_private_key + return if Gitlab::Database.read_only? + + application_settings = Gitlab::CurrentSettings.current_application_settings + application_settings.with_lock do + unless application_settings.lets_encrypt_private_key + application_settings.update(lets_encrypt_private_key: OpenSSL::PKey::RSA.new(4096).to_pem) + end + + application_settings.lets_encrypt_private_key + end + end end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 31e6fc9d8c7..124e34562c1 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -35,8 +35,7 @@ module Gitlab end def token_valid?(token_to_check) - HMACToken.new(actor).token_valid?(token_to_check) || - LegacyRedisDeviseToken.new(actor).token_valid?(token_to_check) + HMACToken.new(actor).token_valid?(token_to_check) end def deploy_key_pushable?(project) @@ -103,44 +102,5 @@ module Gitlab Settings.attr_encrypted_db_key_base.first(16) end end - - # TODO: LegacyRedisDeviseToken and references need to be removed after - # next released milestone - # - class LegacyRedisDeviseToken - TOKEN_LENGTH = 50 - DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins - - def initialize(actor) - @actor = actor - end - - def token_valid?(token_to_check) - Devise.secure_compare(stored_token, token_to_check) - end - - def stored_token - Gitlab::Redis::SharedState.with { |redis| redis.get(state_key) } - end - - # This method exists purely to facilitate legacy testing to ensure the - # same redis key is used. - # - def store_new_token(expiry_time_in_ms = DEFAULT_EXPIRY_TIME) - Gitlab::Redis::SharedState.with do |redis| - new_token = Devise.friendly_token(TOKEN_LENGTH) - redis.set(state_key, new_token, px: expiry_time_in_ms) - new_token - end - end - - private - - attr_reader :actor - - def state_key - "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" - end - end end end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb new file mode 100644 index 00000000000..25e40c70230 --- /dev/null +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'puma/state_file' + +module Gitlab + module Metrics + module Samplers + class PumaSampler < BaseSampler + def metrics + @metrics ||= init_metrics + end + + def init_metrics + { + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), + puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + } + end + + def sample + json_stats = puma_stats + return unless json_stats + + stats = JSON.parse(json_stats) + + if cluster?(stats) + sample_cluster(stats) + else + sample_single_worker(stats) + end + end + + private + + def puma_stats + Puma.stats + rescue NoMethodError + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + nil + end + + def sample_cluster(stats) + set_master_metrics(stats) + + stats['worker_status'].each do |worker| + last_status = worker['last_status'] + labels = { worker: "worker_#{worker['index']}" } + + metrics[:puma_phase].set(labels, worker['phase']) + set_worker_metrics(last_status, labels) if last_status.present? + end + end + + def sample_single_worker(stats) + metrics[:puma_workers].set({}, 1) + metrics[:puma_running_workers].set({}, 1) + + set_worker_metrics(stats) + end + + def cluster?(stats) + stats['worker_status'].present? + end + + def set_master_metrics(stats) + labels = { worker: "master" } + + metrics[:puma_workers].set(labels, stats['workers']) + metrics[:puma_running_workers].set(labels, stats['booted_workers']) + metrics[:puma_stale_workers].set(labels, stats['old_workers']) + metrics[:puma_phase].set(labels, stats['phase']) + end + + def set_worker_metrics(stats, labels = {}) + metrics[:puma_running].set(labels, stats['running']) + metrics[:puma_queued_connections].set(labels, stats['backlog']) + metrics[:puma_active_connections].set(labels, stats['max_threads'] - stats['pool_capacity']) + metrics[:puma_pool_capacity].set(labels, stats['pool_capacity']) + metrics[:puma_max_threads].set(labels, stats['max_threads']) + metrics[:puma_idle_threads].set(labels, stats['running'] + stats['pool_capacity'] - stats['max_threads']) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 18a69321905..4d9c43f37e7 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -23,25 +23,32 @@ module Gitlab end def init_metrics - metrics = {} - metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics = { + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + } + GC.stat.keys.each do |key| metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) - metrics end def sample start_time = System.monotonic_time - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - + metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) @@ -61,6 +68,14 @@ module Gitlab metrics[:total_time].increment(labels, GC::Profiler.total_time) end + def set_memory_usage_metrics + memory_usage = System.memory_usage + memory_labels = labels.merge(worker_label) + + metrics[:memory_bytes].set(memory_labels, memory_usage) + metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) + end + def worker_label return {} unless defined?(Unicorn::Worker) diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..9af7e0afed4 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,16 +4,16 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler - def initialize(interval) - super(interval) + def metrics + @metrics ||= init_metrics end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) - end - - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') + } end def enabled? @@ -23,14 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('tcp', addr, stats) end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('unix', addr, stats) end + + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +38,13 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def set_unicorn_connection_metrics(type, addr, stats) + labels = { socket_type: type, socket_address: addr } + + metrics[:unicorn_active_connections].set(labels, stats.active) + metrics[:unicorn_queued_connections].set(labels, stats.queued) + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +52,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count + end end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 426496855e3..33c0de91c11 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -23,6 +23,22 @@ module Gitlab def self.file_descriptor_count Dir.glob('/proc/self/fd/*').length end + + def self.max_open_file_descriptors + match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) + + return unless match && match[1] + + match[1].to_i + end + + def self.process_start_time + fields = File.read('/proc/self/stat').split + + # fields[21] is linux proc stat field "(22) starttime". + # The value is expressed in clock ticks, divide by clock ticks for seconds. + ( fields[21].to_i || 0 ) / clk_tck + end else def self.memory_usage 0.0 @@ -31,6 +47,14 @@ module Gitlab def self.file_descriptor_count 0 end + + def self.max_open_file_descriptors + 0 + end + + def self.process_start_time + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -59,6 +83,10 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end + + def self.clk_tck + @clk_tck ||= `getconf CLK_TCK`.to_i + end end end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index e0ac9eec1f2..2a2083ebae0 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -36,12 +36,25 @@ module Gitlab hash_arguments = provider['args'].merge(provider_defaults(provider)) # A Hash from the configuration will be passed as is. - provider_arguments << hash_arguments.symbolize_keys + provider_arguments << normalize_hash_arguments(hash_arguments) end provider_arguments end + def normalize_hash_arguments(args) + args.symbolize_keys! + + # Rails 5.1 deprecated the use of string names in the middleware + # (https://github.com/rails/rails/commit/83b767ce), so we need to + # pass in the actual class to Devise. + if args[:strategy_class].is_a?(String) + args[:strategy_class] = args[:strategy_class].constantize + end + + args + end + def provider_defaults(provider) case provider['name'] when 'cas3' diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index aa2c1ac9cef..a07b1246bee 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -58,7 +58,6 @@ module Gitlab uploads users v2 - visual-review-toolbar.js ].freeze # This list should contain all words following `/*namespace_id/:project_id` in diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb new file mode 100644 index 00000000000..3885a9934d5 --- /dev/null +++ b/lib/gitlab/phabricator_import.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + BaseError = Class.new(StandardError) + + def self.available? + Feature.enabled?(:phabricator_import) && + Gitlab::CurrentSettings.import_sources.include?('phabricator') + end + end +end diff --git a/lib/gitlab/phabricator_import/base_worker.rb b/lib/gitlab/phabricator_import/base_worker.rb new file mode 100644 index 00000000000..b69c65e78f8 --- /dev/null +++ b/lib/gitlab/phabricator_import/base_worker.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +# All workers within a Phabricator import should inherit from this worker and +# implement the `#import` method. The jobs should then be scheduled using the +# `.schedule` class method instead of `.perform_async` +# +# Doing this makes sure that only one job of that type is running at the same time +# for a certain project. This will avoid deadlocks. When a job is already running +# we'll wait for it for 10 times 5 seconds to restart. If the running job hasn't +# finished, by then, we'll retry in 30 seconds. +# +# It also makes sure that we keep the import state of the project up to date: +# - It keeps track of the jobs so we know how many jobs are running for the +# project +# - It refreshes the import jid, so it doesn't get cleaned up by the +# `StuckImportJobsWorker` +# - It marks the import as failed if a job failed to many times +# - It marks the import as finished when all remaining jobs are done +module Gitlab + module PhabricatorImport + class BaseWorker + include ApplicationWorker + include ProjectImportOptions # This marks the project as failed after too many tries + include Gitlab::ExclusiveLeaseHelpers + + class << self + def schedule(project_id, *args) + perform_async(project_id, *args) + add_job(project_id) + end + + def add_job(project_id) + worker_state(project_id).add_job + end + + def remove_job(project_id) + worker_state(project_id).remove_job + end + + def worker_state(project_id) + Gitlab::PhabricatorImport::WorkerState.new(project_id) + end + end + + def perform(project_id, *args) + in_lock("#{self.class.name.underscore}/#{project_id}/#{args}", ttl: 2.hours, sleep_sec: 5.seconds) do + project = Project.find_by_id(project_id) + next unless project + + # Bail if the import job already failed + next unless project.import_state&.in_progress? + + project.import_state.refresh_jid_expiration + + import(project, *args) + + # If this is the last running job, finish the import + project.after_import if self.class.worker_state(project_id).running_count < 2 + + self.class.remove_job(project_id) + end + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + # Reschedule a job if there was already a running one + # Running them at the same time could cause a deadlock updating the same + # resource + self.class.perform_in(30.seconds, project_id, *args) + end + + private + + def import(project, *args) + importer_class.new(project, *args).execute + end + + def importer_class + raise NotImplementedError, "Implement `#{__method__}` on #{self.class}" + end + end + end +end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb new file mode 100644 index 00000000000..fa8b37b20ca --- /dev/null +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Cache + class Map + def initialize(project) + @project = project + end + + def get_gitlab_model(phabricator_id) + cached_info = get(phabricator_id) + return unless cached_info[:classname] && cached_info[:database_id] + + cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + end + + def set_gitlab_model(object, phabricator_id) + set(object.class, object.id, phabricator_id) + end + + private + + attr_reader :project + + def set(klass_name, object_id, phabricator_id) + key = cache_key_for_phabricator_id(phabricator_id) + + redis.with do |r| + r.multi do |multi| + multi.mapped_hmset(key, + { classname: klass_name, database_id: object_id }) + multi.expire(key, timeout) + end + end + end + + def get(phabricator_id) + key = cache_key_for_phabricator_id(phabricator_id) + + redis.with do |r| + r.pipelined do |pipe| + # Extend the TTL when a key was + pipe.expire(key, timeout) + pipe.mapped_hmget(key, :classname, :database_id) + end.last + end + end + + def cache_key_for_phabricator_id(phabricator_id) + "#{Redis::Cache::CACHE_NAMESPACE}/phabricator-import/#{project.id}/#{phabricator_id}" + end + + def redis + Gitlab::Redis::Cache + end + + def timeout + # Setting the timeout to the same one as we do for clearing stuck jobs + # this makes sure all cache is available while the import is running. + StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit.rb b/lib/gitlab/phabricator_import/conduit.rb new file mode 100644 index 00000000000..4c64d737389 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + ApiError = Class.new(Gitlab::PhabricatorImport::BaseError) + ResponseError = Class.new(ApiError) + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb new file mode 100644 index 00000000000..4469a3f5849 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/client.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Client + def initialize(phabricator_url, api_token) + @phabricator_url = phabricator_url + @api_token = api_token + end + + def get(path, params: {}) + response = Gitlab::HTTP.get(build_url(path), body: build_params(params), headers: headers) + Response.parse!(response) + rescue *Gitlab::HTTP::HTTP_ERRORS => e + # Wrap all errors from the API into an API-error. + raise ApiError.new(e) + end + + private + + attr_reader :phabricator_url, :api_token + + def headers + { "Accept" => 'application/json' } + end + + def build_url(path) + URI.join(phabricator_url, '/api/', path).to_s + end + + def build_params(params) + params = params.dup + params.compact! + params.reverse_merge!("api.token" => api_token) + + CGI.unescape(params.to_query) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/maniphest.rb b/lib/gitlab/phabricator_import/conduit/maniphest.rb new file mode 100644 index 00000000000..848b71e49e7 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/maniphest.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Maniphest + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def tasks(after: nil) + TasksResponse.new(get_tasks(after)) + end + + private + + def get_tasks(after) + client.get('maniphest.search', + params: { + after: after, + attachments: { projects: 1, subscribers: 1, columns: 1 } + }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/pagination.rb b/lib/gitlab/phabricator_import/conduit/pagination.rb new file mode 100644 index 00000000000..5f54cccdbc8 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/pagination.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Pagination + def initialize(cursor_json) + @cursor_json = cursor_json + end + + def has_next_page? + next_page.present? + end + + def next_page + cursor_json["after"] + end + + private + + attr_reader :cursor_json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb new file mode 100644 index 00000000000..6053ecfbd5e --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/response.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Response + def self.parse!(http_response) + unless http_response.success? + raise Gitlab::PhabricatorImport::Conduit::ResponseError, + "Phabricator responded with #{http_response.status}" + end + + response = new(JSON.parse(http_response.body)) + + unless response.success? + raise ResponseError, + "Phabricator Error: #{response.error_code}: #{response.error_info}" + end + + response + rescue JSON::JSONError => e + raise ResponseError.new(e) + end + + def initialize(json) + @json = json + end + + def success? + error_code.nil? + end + + def error_code + json['error_code'] + end + + def error_info + json['error_info'] + end + + def data + json_result&.fetch('data') + end + + def pagination + return unless cursor_info = json_result&.fetch('cursor') + + @pagination ||= Pagination.new(cursor_info) + end + + private + + attr_reader :json + + def json_result + json['result'] + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/tasks_response.rb b/lib/gitlab/phabricator_import/conduit/tasks_response.rb new file mode 100644 index 00000000000..cbcf7259fb2 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/tasks_response.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class TasksResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + delegate :pagination, to: :conduit_response + + def tasks + @tasks ||= conduit_response.data.map do |task_json| + Gitlab::PhabricatorImport::Representation::Task.new(task_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/import_tasks_worker.rb b/lib/gitlab/phabricator_import/import_tasks_worker.rb new file mode 100644 index 00000000000..c36954a8d41 --- /dev/null +++ b/lib/gitlab/phabricator_import/import_tasks_worker.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + class ImportTasksWorker < BaseWorker + def importer_class + Gitlab::PhabricatorImport::Issues::Importer + end + end + end +end diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb new file mode 100644 index 00000000000..c1797f4027e --- /dev/null +++ b/lib/gitlab/phabricator_import/importer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class Importer + def self.async? + true + end + + def self.imports_repository? + # This does not really import a repository, but we want to skip all + # repository related tasks in the `Projects::ImportService` + true + end + + def initialize(project) + @project = project + end + + def execute + Gitlab::Import::SetAsyncJid.set_jid(project) + schedule_first_tasks_page + + true + rescue => e + fail_import(e.message) + + false + end + + private + + attr_reader :project + + def schedule_first_tasks_page + ImportTasksWorker.schedule(project.id) + end + + def fail_import(message) + project.import_state.mark_as_failed(message) + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb new file mode 100644 index 00000000000..a58438452ff --- /dev/null +++ b/lib/gitlab/phabricator_import/issues/importer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Issues + class Importer + def initialize(project, after = nil) + @project, @after = project, after + end + + def execute + schedule_next_batch + + tasks_response.tasks.each do |task| + TaskImporter.new(project, task).execute + end + end + + private + + attr_reader :project, :after + + def schedule_next_batch + return unless tasks_response.pagination.has_next_page? + + Gitlab::PhabricatorImport::ImportTasksWorker + .schedule(project.id, tasks_response.pagination.next_page) + end + + def tasks_response + @tasks_response ||= client.tasks(after: after) + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::Maniphest + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb new file mode 100644 index 00000000000..40d4392cbc1 --- /dev/null +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Issues + class TaskImporter + def initialize(project, task) + @project, @task = project, task + end + + def execute + # TODO: get the user from the project namespace from the username loaded by Phab-id + # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 + issue.author = User.ghost + + # TODO: Reformat the description with attachments, escaping accidental + # links and add attachments + # https://gitlab.com/gitlab-org/gitlab-ce/issues/60603 + issue.assign_attributes(task.issue_attributes) + + save! + + issue + end + + private + + attr_reader :project, :task + + def save! + # Just avoiding an extra redis call, we've already updated the expiry + # when reading the id from the map + was_persisted = issue.persisted? + + issue.save! if issue.changed? + + object_map.set_gitlab_model(issue, task.phabricator_id) unless was_persisted + end + + def issue + @issue ||= find_issue_by_phabricator_id(task.phabricator_id) || + project.issues.new + end + + def find_issue_by_phabricator_id(phabricator_id) + object_map.get_gitlab_model(phabricator_id) + end + + def object_map + Gitlab::PhabricatorImport::Cache::Map.new(project) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb new file mode 100644 index 00000000000..b37a5b44980 --- /dev/null +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class ProjectCreator + def initialize(current_user, params) + @current_user = current_user + @params = params.dup + end + + def execute + return unless import_url.present? && api_token.present? + + project = Projects::CreateService.new(current_user, create_params).execute + return project unless project.persisted? + + project.project_feature.update!(project_feature_attributes) + + project + end + + private + + attr_reader :current_user, :params + + def create_params + { + name: project_name, + path: project_path, + namespace_id: namespace_id, + import_type: 'phabricator', + import_url: Project::UNKNOWN_IMPORT_URL, + import_data: import_data + } + end + + def project_name + params[:name] + end + + def project_path + params[:path] + end + + def namespace_id + params[:namespace_id] || current_user.namespace_id + end + + def import_url + params[:phabricator_server_url] + end + + def api_token + params[:api_token] + end + + def project_feature_attributes + @project_features_attributes ||= begin + # everything disabled except for issues + ProjectFeature::FEATURES.map do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) + end + end + + def import_data + { + data: { + phabricator_url: import_url + }, + credentials: { + api_token: params.fetch(:api_token) + } + } + end + end + end +end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb new file mode 100644 index 00000000000..6aedc71b626 --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Representation + class Task + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def issue_attributes + @issue_attributes ||= { + title: issue_title, + description: issue_description, + state: issue_state, + created_at: issue_created_at, + closed_at: issue_closed_at + } + end + + private + + attr_reader :json + + def issue_title + # The 255 limit is the validation we impose on the Issue title in + # Issuable + @issue_title ||= json['fields']['name'].truncate(255) + end + + def issue_description + json['fields']['description']['raw'] + end + + def issue_state + issue_closed_at.present? ? :closed : :opened + end + + def issue_created_at + return unless json['fields']['dateCreated'] + + @issue_created_at ||= cast_datetime(json['fields']['dateCreated']) + end + + def issue_closed_at + return unless json['fields']['dateClosed'] + + @issue_closed_at ||= cast_datetime(json['fields']['dateClosed']) + end + + def cast_datetime(value) + Time.at(value.to_i) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb new file mode 100644 index 00000000000..38829e34509 --- /dev/null +++ b/lib/gitlab/phabricator_import/worker_state.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + class WorkerState + def initialize(project_id) + @project_id = project_id + end + + def add_job + redis.with do |r| + r.pipelined do |pipe| + pipe.incr(all_jobs_key) + pipe.expire(all_jobs_key, timeout) + end + end + end + + def remove_job + redis.with do |r| + r.decr(all_jobs_key) + end + end + + def running_count + redis.with { |r| r.get(all_jobs_key) }.to_i + end + + private + + attr_reader :project_id + + def redis + Gitlab::Redis::SharedState + end + + def all_jobs_key + @all_jobs_key ||= "phabricator-import/jobs/project-#{project_id}/job-count" + end + + def timeout + # Make sure we get rid of all the information after a job is marked + # as failed/succeeded + StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 78337518988..0f3b97e2317 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -138,6 +138,12 @@ module Gitlab project end + def filter_milestones_by_project(milestones) + return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) + + milestones.where(project_id: project.id) # rubocop: disable CodeReuse/ActiveRecord + end + def repository_project_ref @repository_project_ref ||= repository_ref || project.default_branch end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index dca09aef47d..9cc21129547 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -5,8 +5,7 @@ module Gitlab module QueryVariables def self.call(environment) deployment_platform = environment.deployment_platform - namespace = deployment_platform&.namespace_for(environment.project) || - deployment_platform&.actual_namespace || '' + namespace = deployment_platform&.kubernetes_namespace_for(environment.project) || '' { ci_environment_slug: environment.slug, diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 1cd158db2b0..e1579cfddc0 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -102,7 +102,7 @@ module Gitlab @updates[:milestone_id] = nil end - desc _('Copy labels and milestone from other issue or merge request') + desc _('Copy labels and milestone from other issue or merge request in this project') explanation do |source_issuable| _("Copy labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end diff --git a/lib/gitlab/rack_timeout_observer.rb b/lib/gitlab/rack_timeout_observer.rb new file mode 100644 index 00000000000..80d3f7dea60 --- /dev/null +++ b/lib/gitlab/rack_timeout_observer.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + class RackTimeoutObserver + def initialize + @counter = Gitlab::Metrics.counter(:rack_state_total, 'Number of requests in a given rack state') + end + + # returns the Proc to be used as the observer callback block + def callback + method(:log_timeout_exception) + end + + private + + def log_timeout_exception(env) + info = env[::Rack::Timeout::ENV_INFO_KEY] + return unless info + + @counter.increment(labels(info, env)) + end + + def labels(info, env) + params = controller_params(env) || grape_params(env) || {} + + { + controller: params['controller'], + action: params['action'], + route: params['route'], + state: info.state + } + end + + def controller_params(env) + env['action_dispatch.request.parameters'] + end + + def grape_params(env) + endpoint = env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + return unless route + + { 'route' => route } + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 4a097a00101..7c1e6b1baff 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -103,9 +103,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def milestones - milestones = Milestone.where(project_id: project_ids_relation) - milestones = milestones.search(query) - milestones.reorder('milestones.updated_at DESC') + milestones = Milestone.search(query) + + milestones = filter_milestones_by_project(milestones) + + milestones.reorder('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord @@ -123,6 +125,26 @@ module Gitlab 'projects' end + # Filter milestones by authorized projects. + # For performance reasons project_id is being plucked + # to be used on a smaller query. + # + # rubocop: disable CodeReuse/ActiveRecord + def filter_milestones_by_project(milestones) + project_ids = + milestones.where(project_id: project_ids_relation) + .select(:project_id).distinct + .pluck(:project_id) + + return Milestone.none if project_ids.nil? + + authorized_project_ids_relation = + Project.where(id: project_ids).ids_with_milestone_available_for(current_user) + + milestones.where(project_id: authorized_project_ids_relation) + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 641ba70ef83..9a8df719827 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -8,38 +8,68 @@ module Gitlab BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, ports: [], schemes: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false) - return true if url.nil? + # Validates the given url according to the constraints specified by arguments. + # + # ports - Raises error if the given URL port does is not between given ports. + # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is true. + # allow_local_network - Raises error if URL resolves to a link-local address and argument is true. + # ascii_only - Raises error if URL has unicode characters and argument is true. + # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. + # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # + # Returns an array with [<uri>, <original-hostname>]. + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/ParameterLists + def validate!( + url, + ports: [], + schemes: [], + allow_localhost: false, + allow_local_network: true, + ascii_only: false, + enforce_user: false, + enforce_sanitization: false, + dns_rebind_protection: true) + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/ParameterLists + + return [nil, nil] if url.nil? # Param url can be a string, URI or Addressable::URI uri = parse_url(url) validate_html_tags!(uri) if enforce_sanitization - # Allow imports from the GitLab instance itself but only from the configured ports - return true if internal?(uri) - + hostname = uri.hostname port = get_port(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(uri.hostname) - validate_unicode_restriction!(uri) if ascii_only + + unless internal?(uri) + validate_scheme!(uri.scheme, schemes) + validate_port!(port, ports) if ports.any? + validate_user!(uri.user) if enforce_user + validate_hostname!(hostname) + validate_unicode_restriction!(uri) if ascii_only + end begin - addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr| + addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError - return true + return [uri, nil] end + protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + + # Allow url from the GitLab instance itself but only for the configured hostname and ports + return protected_uri_with_hostname if internal?(uri) + validate_localhost!(addrs_info) unless allow_localhost validate_loopback!(addrs_info) unless allow_localhost validate_local_network!(addrs_info) unless allow_local_network validate_link_local!(addrs_info) unless allow_local_network - true + protected_uri_with_hostname end def blocked_url?(*args) @@ -52,6 +82,25 @@ module Gitlab private + # Returns the given URI with IP address as hostname and the original hostname respectively + # in an Array. + # + # It checks whether the resolved IP address matches with the hostname. If not, it changes + # the hostname to the resolved IP address. + # + # The original hostname is used to validate the SSL, given in that scenario + # we'll be making the request to the IP address, instead of using the hostname. + def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + address = addrs_info.first + ip_address = address&.ip_address + + return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname + + uri = uri.dup + uri.hostname = ip_address + [uri, hostname] + end + def get_port(uri) uri.port || uri.default_port end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 880712de5fe..215454fe63c 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -47,6 +47,10 @@ module Gitlab @credentials ||= { user: @url.user.presence, password: @url.password.presence } end + def user + credentials[:user] + end + def full_url @full_url ||= generate_full_url.to_s end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb new file mode 100644 index 00000000000..24d8eac200c --- /dev/null +++ b/lib/quality/test_level.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Quality + class TestLevel + UnknownTestLevelError = Class.new(StandardError) + + TEST_LEVEL_FOLDERS = { + unit: %w[ + bin + config + db + dependencies + factories + finders + frontend + graphql + helpers + initializers + javascripts + lib + migrations + models + policies + presenters + rack_servers + routing + rubocop + serializers + services + sidekiq + tasks + uploaders + validators + views + workers + elastic_integration + ], + integration: %w[ + controllers + mailers + requests + ], + system: ['features'] + }.freeze + + attr_reader :prefix + + def initialize(prefix = nil) + @prefix = prefix + @patterns = {} + @regexps = {} + end + + def pattern(level) + @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb".freeze + end + + def regexp(level) + @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze + end + + def level_for(file_path) + case file_path + when regexp(:unit) + :unit + when regexp(:integration) + :integration + when regexp(:system) + :system + else + raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}." + end + end + end +end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 7a42e4e92a0..a07ae3a418a 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -10,9 +10,15 @@ namespace :gitlab do rake:assets:precompile webpack:compile gitlab:assets:fix_urls + gitlab:assets:compile_vrt ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) end + desc 'GitLab | Assets | Compile visual review toolbar' + task :compile_vrt do + system 'yarn', 'webpack-vrt' + end + desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ee3ef9dad6e..487808a7baa 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -51,9 +51,6 @@ namespace :gitlab do end end - # (Re)create hooks - Rake::Task['gitlab:shell:create_hooks'].invoke - Gitlab::Shell.ensure_secret_token! end @@ -78,15 +75,6 @@ namespace :gitlab do end end end - - desc 'Create or repair repository hooks symlink' - task create_hooks: :gitlab_environment do - warn_user_is_not_gitlab - - puts 'Creating/Repairing hooks symlinks for all repositories' - system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) - puts 'done'.color(:green) - end end def setup diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 2eddcb3c777..c881ad4cf12 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -1,7 +1,32 @@ +# frozen_string_literal: true + +return if Rails.env.production? + Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') namespace :spec do - desc 'GitLab | Rspec | Run request specs' + desc 'GitLab | RSpec | Run unit tests' + RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:unit) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run integration tests' + RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:integration) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run system tests' + RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:system) + t.rspec_opts = args[:rspec_opts] + end + + desc '[Deprecated] Use the "bin/rspec --tag api" instead' task :api do cmds = [ %w(rake gitlab:setup), @@ -10,7 +35,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run feature specs' + desc '[Deprecated] Use the "spec:system" task instead' task :feature do cmds = [ %w(rake gitlab:setup), @@ -19,7 +44,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run model specs' + desc '[Deprecated] Use "bin/rspec spec/models" instead' task :models do cmds = [ %w(rake gitlab:setup), @@ -28,7 +53,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run service specs' + desc '[Deprecated] Use "bin/rspec spec/services" instead' task :services do cmds = [ %w(rake gitlab:setup), @@ -37,7 +62,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run lib specs' + desc '[Deprecated] Use "bin/rspec spec/lib" instead' task :lib do cmds = [ %w(rake gitlab:setup), @@ -45,15 +70,6 @@ namespace :spec do ] run_commands(cmds) end - - desc 'GitLab | Rspec | Run other specs' - task :other do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) - ] - run_commands(cmds) - end end desc "GitLab | Run specs" |