diff options
Diffstat (limited to 'lib')
211 files changed, 5377 insertions, 2761 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 60ae5e6b9a2..ae13c248171 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -53,7 +53,10 @@ module API put ':id/access_requests/:user_id/approve' do source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute + access_requester = source.requesters.find_by!(user_id: params[:user_id]) + member = ::Members::ApproveAccessRequestService + .new(current_user, declared_params) + .execute(access_requester) status :created present member, with: Entities::Member @@ -70,8 +73,7 @@ module API member = source.requesters.find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do - ::Members::DestroyService.new(source, current_user, params) - .execute(:requesters) + ::Members::DestroyService.new(current_user).execute(member) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 754549f72f0..62ffebeacb0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -108,6 +108,7 @@ module API mount ::API::AccessRequests mount ::API::Applications mount ::API::AwardEmoji + mount ::API::Badges mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages @@ -120,6 +121,7 @@ module API mount ::API::Events mount ::API::Features mount ::API::Files + mount ::API::GroupBoards mount ::API::Groups mount ::API::GroupMilestones mount ::API::Internal @@ -134,10 +136,12 @@ module API mount ::API::MergeRequests mount ::API::Namespaces mount ::API::Notes + mount ::API::Discussions mount ::API::NotificationSettings mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/badges.rb b/lib/api/badges.rb new file mode 100644 index 00000000000..334948b2995 --- /dev/null +++ b/lib/api/badges.rb @@ -0,0 +1,134 @@ +module API + class Badges < Grape::API + include PaginationParams + + before { authenticate_non_get! } + + helpers ::API::Helpers::BadgesHelpers + + helpers do + def find_source_if_admin(source_type) + source = find_source(source_type, params[:id]) + + authorize_admin_source!(source_type, source) + + source + end + end + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The ID of a #{source_type}" + end + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Gets a list of #{source_type} badges viewable by the authenticated user." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + use :pagination + end + get ":id/badges" do + source = find_source(source_type, params[:id]) + + present_badges(source, paginate(source.badges)) + end + + desc "Preview a badge from a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::BasicBadgeDetails + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + get ":id/badges/render" do + authenticate! + + source = find_source_if_admin(source_type) + + badge = ::Badges::BuildService.new(declared_params(include_missing: false)) + .execute(source) + + if badge.valid? + present_badges(source, badge, with: Entities::BasicBadgeDetails) + else + render_validation_error!(badge) + end + end + + desc "Gets a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + get ":id/badges/:badge_id" do + source = find_source(source_type, params[:id]) + badge = find_badge(source) + + present_badges(source, badge) + end + + desc "Adds a badge to a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + post ":id/badges" do + source = find_source_if_admin(source_type) + + badge = ::Badges::CreateService.new(declared_params(include_missing: false)).execute(source) + + if badge.persisted? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc "Updates a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + optional :link_url, type: String, desc: 'URL of the badge link' + optional :image_url, type: String, desc: 'URL of the badge image' + end + put ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + + badge = ::Badges::UpdateService.new(declared_params(include_missing: false)) + .execute(find_badge(source)) + + if badge.valid? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc 'Removes a badge from a project or group.' do + detail 'This feature was introduced in GitLab 10.6.' + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + delete ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + badge = find_badge(source) + + if badge.is_a?(GroupBadge) && source.is_a?(Project) + error!('To delete a Group badge please use the Group endpoint', 403) + end + + destroy_conditionally!(badge) + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 1794207e29b..13cfba728fa 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -16,6 +16,10 @@ module API render_api_error!('The branch refname is invalid', 400) end end + + params :filter_params do + optional :search, type: String, desc: 'Return list of branches matching the search criteria' + end end params do @@ -27,15 +31,23 @@ module API end params do use :pagination + use :filter_params end get ':id/repository/branches' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') repository = user_project.repository - branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) + + branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names + present( + paginate(::Kaminari.paginate_array(branches)), + with: Entities::Branch, + project: user_project, + merged_branch_names: merged_branch_names + ) end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 3d6e78d2d80..982f45425a3 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -18,25 +18,28 @@ module API optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' optional :path, type: String, desc: 'The file path' + optional :all, type: Boolean, desc: 'Every commit will be returned' use :pagination end get ':id/repository/commits' do path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] + all = params[:all] commits = user_project.repository.commits(ref, path: path, limit: params[:per_page], offset: offset, before: before, - after: after) + after: after, + all: all) commit_count = - if path || before || after - user_project.repository.count_commits(ref: ref, path: path, before: before, after: after) + if all || path || before || after + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all) else # Cacheable commit count. user_project.repository.commit_count_for_ref(ref) diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb new file mode 100644 index 00000000000..6abd575b6ad --- /dev/null +++ b/lib/api/discussions.rb @@ -0,0 +1,195 @@ +module API + class Discussions < Grape::API + include PaginationParams + helpers ::API::Helpers::NotesHelpers + + before { authenticate! } + + NOTEABLE_TYPES = [Issue, Snippet].freeze + + NOTEABLE_TYPES.each do |noteable_type| + parent_type = noteable_type.parent_class.to_s.underscore + noteables_str = noteable_type.to_s.underscore.pluralize + + params do + requires :id, type: String, desc: "The ID of a #{parent_type}" + end + resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a list of #{noteable_type.to_s.downcase} discussions" do + success Entities::Discussion + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination + end + get ":id/#{noteables_str}/:noteable_id/discussions" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + return not_found!("Discussions") unless can?(current_user, noteable_read_ability_name(noteable), noteable) + + notes = noteable.notes + .inc_relations_for_view + .includes(:noteable) + .fresh + + notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + discussions = Kaminari.paginate_array(Discussion.build_collection(notes, noteable)) + + present paginate(discussions), with: Entities::Discussion + end + + desc "Get a single #{noteable_type.to_s.downcase} discussion" do + success Entities::Discussion + end + params do + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + notes = readable_discussion_notes(noteable, params[:discussion_id]) + + if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) + return not_found!("Discussion") + end + + discussion = Discussion.build(notes, noteable) + + present discussion, with: Entities::Discussion + end + + desc "Create a new #{noteable_type.to_s.downcase} discussion" do + success Entities::Discussion + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/discussions" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + opts = { + note: params[:body], + created_at: params[:created_at], + type: 'DiscussionNote', + noteable_type: noteables_str.classify, + noteable_id: noteable.id + } + + note = create_note(noteable, opts) + + if note.valid? + present note.discussion, with: Entities::Discussion + else + bad_request!("Note #{note.errors.messages}") + end + end + + desc "Get comments in a single #{noteable_type.to_s.downcase} discussion" do + success Entities::Discussion + end + params do + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + notes = readable_discussion_notes(noteable, params[:discussion_id]) + + if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) + return not_found!("Notes") + end + + present notes, with: Entities::Note + end + + desc "Add a comment to a #{noteable_type.to_s.downcase} discussion" do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + notes = readable_discussion_notes(noteable, params[:discussion_id]) + + return not_found!("Discussion") if notes.empty? + return bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? + + opts = { + note: params[:body], + type: 'DiscussionNote', + in_reply_to_discussion_id: params[:discussion_id], + created_at: params[:created_at] + } + note = create_note(noteable, opts) + + if note.valid? + present note, with: Entities::Note + else + bad_request!("Note #{note.errors.messages}") + end + end + + desc "Get a comment in a #{noteable_type.to_s.downcase} discussion" do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + get_note(noteable, params[:note_id]) + end + + desc "Edit a comment in a #{noteable_type.to_s.downcase} discussion" do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :body, type: String, desc: 'The content of a note' + end + put ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + update_note(noteable, params[:note_id]) + end + + desc "Delete a comment in a #{noteable_type.to_s.downcase} discussion" do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + delete ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + delete_note(noteable, params[:note_id]) + end + end + end + + helpers do + def readable_discussion_notes(noteable, discussion_id) + notes = noteable.notes + .where(discussion_id: discussion_id) + .inc_relations_for_view + .includes(:noteable) + .fresh + + notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 167878ba600..16147ee90c9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,7 +71,7 @@ module API end class ProjectHook < Hook - expose :project_id, :issues_events + expose :project_id, :issues_events, :confidential_issues_events expose :note_events, :pipeline_events, :wiki_page_events expose :job_events end @@ -91,6 +91,21 @@ module API expose :created_at end + class ProjectExportStatus < ProjectIdentity + include ::API::Helpers::RelatedResourcesHelpers + + expose :export_status + expose :_links, if: lambda { |project, _options| project.export_status == :finished } do + expose :api_url do |project| + expose_url(api_v4_projects_export_download_path(id: project.id)) + end + + expose :web_url do |project| + Gitlab::Routing.url_helpers.download_export_project_url(project) + end + end + end + class ProjectImportStatus < ProjectIdentity expose :import_status @@ -481,6 +496,10 @@ module API expose :id end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class MergeRequestSimple < ProjectEntity expose :title expose :web_url do |merge_request, options| @@ -528,6 +547,7 @@ module API expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch + expose :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) @@ -546,6 +566,42 @@ module API expose :changes_count do |merge_request, _options| merge_request.merge_request_diff.real_size end + + expose :merged_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.merged_by + end + + expose :merged_at do |merge_request, _options| + merge_request.metrics&.merged_at + end + + expose :closed_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.latest_closed_by + end + + expose :closed_at do |merge_request, _options| + merge_request.metrics&.latest_closed_at + end + + expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.latest_build_started_at + end + + expose :latest_build_finished_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.latest_build_finished_at + end + + expose :first_deployed_to_production_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.first_deployed_to_production_at + end + + expose :pipeline, using: Entities::PipelineBasic, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.pipeline + end + + def build_available?(options) + options[:project]&.feature_available?(:builds, options[:current_user]) + end end class MergeRequestChanges < MergeRequest @@ -589,6 +645,7 @@ module API NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze expose :id + expose :type expose :note, as: :body expose :attachment_identifier, as: :attachment expose :author, using: Entities::UserBasic @@ -600,6 +657,12 @@ module API expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) } end + class Discussion < Grape::Entity + expose :id + expose :individual_note?, as: :individual_note + expose :notes, using: Entities::Note + end + class AwardEmoji < Grape::Entity expose :id expose :name @@ -909,10 +972,6 @@ module API expose :filename, :size end - class PipelineBasic < Grape::Entity - expose :id, :sha, :ref, :status - end - class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -1199,5 +1258,23 @@ module API expose :startline expose :project_id end + + class BasicBadgeDetails < Grape::Entity + expose :link_url + expose :image_url + expose :rendered_link_url do |badge, options| + badge.rendered_link_url(options.fetch(:project, nil)) + end + expose :rendered_image_url do |badge, options| + badge.rendered_image_url(options.fetch(:project, nil)) + end + end + + class Badge < BasicBadgeDetails + expose :id + expose :kind do |badge| + badge.type == 'ProjectBadge' ? 'project' : 'group' + end + end end end diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb new file mode 100644 index 00000000000..aa9fff25fc8 --- /dev/null +++ b/lib/api/group_boards.rb @@ -0,0 +1,117 @@ +module API + class GroupBoards < Grape::API + include BoardsResponses + include PaginationParams + + before do + authenticate! + end + + helpers do + def board_parent + user_group + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + segment ':id/boards' do + desc 'Find a group board' do + detail 'This feature was introduced in 10.6' + success ::API::Entities::Board + end + get '/:board_id' do + present board, with: ::API::Entities::Board + end + + desc 'Get all group boards' do + detail 'This feature was introduced in 10.6' + success Entities::Board + end + params do + use :pagination + end + get '/' do + present paginate(board_parent.boards), with: Entities::Board + end + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + desc 'Get the lists of a group board' do + detail 'Does not include backlog and closed lists. This feature was introduced in 10.6' + success Entities::List + end + params do + use :pagination + end + get '/lists' do + present paginate(board_lists), with: Entities::List + end + + desc 'Get a list of a group board' do + detail 'This feature was introduced in 10.6' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + end + get '/lists/:list_id' do + present board_lists.find(params[:list_id]), with: Entities::List + end + + desc 'Create a new board list' do + detail 'This feature was introduced in 10.6' + success Entities::List + end + params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end + post '/lists' do + unless available_labels_for(board_parent).exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + + authorize!(:admin_list, user_group) + + create_list + end + + desc 'Moves a board list to a new position' do + detail 'This feature was introduced in 10.6' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + requires :position, type: Integer, desc: 'The position of the list' + end + put '/lists/:list_id' do + list = board_lists.find(params[:list_id]) + + authorize!(:admin_list, user_group) + + move_list(list) + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 10.6' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_group) + list = board_lists.find(params[:list_id]) + + destroy_list(list) + end + end + end + end +end diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb new file mode 100644 index 00000000000..1f8afbf3c90 --- /dev/null +++ b/lib/api/helpers/badges_helpers.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module BadgesHelpers + include ::API::Helpers::MembersHelpers + + def find_badge(source) + source.badges.find(params[:badge_id]) + end + + def present_badges(source, records, options = {}) + entity_type = options[:with] || Entities::Badge + badge_params = badge_source_params(source).merge(with: entity_type) + + present records, badge_params + end + + def badge_source_params(source) + project = if source.is_a?(Project) + source + else + GroupProjectsFinder.new(group: source, current_user: current_user).execute.first + end + + { project: project } + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index cd59da6fc70..4b564cfdef2 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -111,13 +111,6 @@ module API def gitaly_payload(action) return unless %w[git-receive-pack git-upload-pack].include?(action) - if action == 'git-receive-pack' - return unless Gitlab::GitalyClient.feature_enabled?( - :ssh_receive_pack, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT - ) - end - { repository: repository.gitaly_repository, address: Gitlab::GitalyClient.address(project.repository_storage), diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb new file mode 100644 index 00000000000..cd91df1ecd8 --- /dev/null +++ b/lib/api/helpers/notes_helpers.rb @@ -0,0 +1,76 @@ +module API + module Helpers + module NotesHelpers + def update_note(noteable, note_id) + note = noteable.notes.find(params[:note_id]) + + authorize! :admin_note, note + + opts = { + note: params[:body] + } + parent = noteable_parent(noteable) + project = parent if parent.is_a?(Project) + + note = ::Notes::UpdateService.new(project, current_user, opts).execute(note) + + if note.valid? + present note, with: Entities::Note + else + bad_request!("Failed to save note #{note.errors.messages}") + end + end + + def delete_note(noteable, note_id) + note = noteable.notes.find(note_id) + + authorize! :admin_note, note + + parent = noteable_parent(noteable) + project = parent if parent.is_a?(Project) + destroy_conditionally!(note) do |note| + ::Notes::DestroyService.new(project, current_user).execute(note) + end + end + + def get_note(noteable, note_id) + note = noteable.notes.with_metadata.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) + + if can_read_note + present note, with: Entities::Note + else + not_found!("Note") + end + end + + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore}".to_sym + end + + def find_noteable(parent, noteables_str, noteable_id) + public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend + end + + def noteable_parent(noteable) + public_send("user_#{noteable.class.parent_class.to_s.underscore}") # rubocop:disable GitlabSecurity/PublicSend + end + + def create_note(noteable, opts) + noteables_str = noteable.model_name.to_s.underscore.pluralize + + return not_found!(noteables_str) unless can?(current_user, noteable_read_ability_name(noteable), noteable) + + authorize! :create_note, noteable + + parent = noteable_parent(noteable) + if opts[:created_at] + opts.delete(:created_at) unless current_user.admin? || parent.owner == current_user + end + + project = parent if parent.is_a?(Project) + ::Notes::CreateService.new(project, current_user, opts).execute + end + end + end +end diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index 1f677529b07..7f4d6e58b34 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -15,7 +15,7 @@ module API url_options = Gitlab::Application.routes.default_url_options protocol, host, port = url_options.slice(:protocol, :host, :port).values - URI::HTTP.build(scheme: protocol, host: host, port: port, path: path).to_s + URI::Generic.build(scheme: protocol, host: host, port: port, path: path).to_s end private diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index fbe30192a16..35ac0b4cbca 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -9,16 +9,22 @@ module API Gitlab::CurrentSettings.runners_registration_token) end - def get_runner_version_from_params - return unless params['info'].present? + def authenticate_runner! + forbidden! unless current_runner - attributes_for_keys(%w(name version revision platform architecture), params['info']) + current_runner + .update_cached_info(get_runner_details_from_request) end - def authenticate_runner! - forbidden! unless current_runner + def get_runner_details_from_request + return get_runner_ip unless params['info'].present? + + attributes_for_keys(%w(name version revision platform architecture), params['info']) + .merge(get_runner_ip) + end - current_runner.update_cached_info(get_runner_version_from_params) + def get_runner_ip + { ip_address: request.ip } end def current_runner diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b6c278c89d0..f74b3b26802 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -32,6 +32,8 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 2a8fa7659bf..47e5eeab31d 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -2,20 +2,28 @@ module API class JobArtifacts < Grape::API before { authenticate_non_get! } + # EE::API::JobArtifacts would override the following helpers + helpers do + def authorize_download_artifacts! + authorize_read_builds! + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Download the artifacts file from a job' do + desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.10' end params do requires :ref_name, type: String, desc: 'The ref from repository' requires :job, type: String, desc: 'The name for the job' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do - authorize_read_builds! + authorize_download_artifacts! builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) @@ -23,14 +31,15 @@ module API present_artifacts!(latest_build.artifacts_file) end - desc 'Download the artifacts file from a job' do + desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.5' end params do requires :job_id, type: Integer, desc: 'The ID of a job' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! + authorize_download_artifacts! build = find_build!(params[:job_id]) diff --git a/lib/api/members.rb b/lib/api/members.rb index bc1de37284a..8b12986d09e 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -81,12 +81,16 @@ module API source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) - member = source.members.find_by!(user_id: params.delete(:user_id)) + member = source.members.find_by!(user_id: params[:user_id]) + updated_member = + ::Members::UpdateService + .new(current_user, declared_params(include_missing: false)) + .execute(member) - if member.update_attributes(declared_params(include_missing: false)) - present member, with: Entities::Member + if updated_member.valid? + present updated_member, with: Entities::Member else - render_validation_error!(member) + render_validation_error!(updated_member) end end @@ -99,7 +103,7 @@ module API member = source.members.find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do - ::Members::DestroyService.new(source, current_user, declared_params).execute + ::Members::DestroyService.new(current_user).execute(member) end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 719afa09295..3264a26f7d2 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -6,6 +6,32 @@ module API helpers ::Gitlab::IssuableMetadata + # EE::API::MergeRequests would override the following helpers + helpers do + params :optional_params_ee do + end + + params :merge_params_ee do + end + + def update_merge_request_ee(merge_request) + end + end + + def self.update_params_at_least_one_of + %i[ + assignee_id + description + labels + milestone_id + remove_source_branch + state_event + target_branch + title + discussion_locked + ] + end + helpers do def find_merge_requests(args = {}) args = declared_params.merge(args) @@ -31,6 +57,12 @@ module API mr.all_pipelines end + def check_sha_param!(params, merge_request) + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) + end + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', desc: 'Return opened, closed, merged, or all merge requests' @@ -42,12 +74,16 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' + optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' optional :search, type: String, desc: 'Search merge requests for text present in the title or description' use :pagination end @@ -102,16 +138,15 @@ module API render_api_error!(errors, 400) end - params :optional_params_ce do + params :optional_params do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - end + optional :allow_maintainer_to_push, type: Boolean, desc: 'Whether a maintainer of the target project can push to the source project' - params :optional_params do - use :optional_params_ce + use :optional_params_ee end end @@ -220,7 +255,7 @@ module API get ':id/merge_requests/:merge_request_iid/changes' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequestChanges, current_user: current_user + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user, project: user_project end desc 'Get the merge request pipelines' do @@ -236,18 +271,6 @@ module API success Entities::MergeRequest end params do - # CE - at_least_one_of_ce = [ - :assignee_id, - :description, - :labels, - :milestone_id, - :remove_source_branch, - :state_event, - :target_branch, - :title, - :discussion_locked - ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], @@ -255,7 +278,7 @@ module API optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' use :optional_params - at_least_one_of(*at_least_one_of_ce) + at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) end put ':id/merge_requests/:merge_request_iid' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318') @@ -278,13 +301,14 @@ module API success Entities::MergeRequest end params do - # CE optional :merge_commit_message, type: String, desc: 'Custom merge commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' optional :merge_when_pipeline_succeeds, type: Boolean, desc: 'When true, this merge request will be merged when the pipeline succeeds' optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + + use :merge_params_ee end put ':id/merge_requests/:merge_request_iid/merge' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') @@ -300,9 +324,9 @@ module API render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) - if params[:sha] && merge_request.diff_head_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) - end + check_sha_param!(params, merge_request) + + update_merge_request_ee(merge_request) merge_params = { commit_message: params[:merge_commit_message], diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 3588dc85c9e..69f1df6b341 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,19 +1,23 @@ module API class Notes < Grape::API include PaginationParams + helpers ::API::Helpers::NotesHelpers before { authenticate! } NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - NOTEABLE_TYPES.each do |noteable_type| + NOTEABLE_TYPES.each do |noteable_type| + parent_type = noteable_type.parent_class.to_s.underscore + noteables_str = noteable_type.to_s.underscore.pluralize + + params do + requires :id, type: String, desc: "The ID of a #{parent_type}" + end + resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do noteables_str = noteable_type.to_s.underscore.pluralize - desc 'Get a list of project +noteable+ notes' do + desc "Get a list of #{noteable_type.to_s.downcase} notes" do success Entities::Note end params do @@ -25,7 +29,7 @@ module API use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = find_project_noteable(noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed @@ -46,7 +50,7 @@ module API end end - desc 'Get a single +noteable+ note' do + desc "Get a single #{noteable_type.to_s.downcase} note" do success Entities::Note end params do @@ -54,18 +58,11 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = find_project_noteable(noteables_str, params[:noteable_id]) - note = noteable.notes.with_metadata.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) - - if can_read_note - present note, with: Entities::Note - else - not_found!("Note") - end + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + get_note(noteable, params[:note_id]) end - desc 'Create a new +noteable+ note' do + desc "Create a new #{noteable_type.to_s.downcase} note" do success Entities::Note end params do @@ -74,34 +71,25 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_str}/:noteable_id/notes" do - noteable = find_project_noteable(noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) opts = { note: params[:body], noteable_type: noteables_str.classify, - noteable_id: noteable.id + noteable_id: noteable.id, + created_at: params[:created_at] } - if can?(current_user, noteable_read_ability_name(noteable), noteable) - authorize! :create_note, noteable + note = create_note(noteable, opts) - if params[:created_at] && (current_user.admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end - - note = ::Notes::CreateService.new(user_project, current_user, opts).execute - - if note.valid? - present note, with: Entities.const_get(note.class.name) - else - not_found!("Note #{note.errors.messages}") - end + if note.valid? + present note, with: Entities.const_get(note.class.name) else - not_found!("Note") + bad_request!("Note #{note.errors.messages}") end end - desc 'Update an existing +noteable+ note' do + desc "Update an existing #{noteable_type.to_s.downcase} note" do success Entities::Note end params do @@ -110,24 +98,12 @@ module API requires :body, type: String, desc: 'The content of a note' end put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - note = user_project.notes.find(params[:note_id]) + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - authorize! :admin_note, note - - opts = { - note: params[:body] - } - - note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) - - if note.valid? - present note, with: Entities::Note - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end + update_note(noteable, params[:note_id]) end - desc 'Delete a +noteable+ note' do + desc "Delete a #{noteable_type.to_s.downcase} note" do success Entities::Note end params do @@ -135,25 +111,11 @@ module API requires :note_id, type: Integer, desc: 'The ID of a note' end delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - note = user_project.notes.find(params[:note_id]) - - authorize! :admin_note, note + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - destroy_conditionally!(note) do |note| - ::Notes::DestroyService.new(user_project, current_user).execute(note) - end + delete_note(noteable, params[:note_id]) end end end - - helpers do - def find_project_noteable(noteables_str, noteable_id) - public_send("find_project_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend - end - - def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore}".to_sym - end - end end end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb new file mode 100644 index 00000000000..6ec2626df1a --- /dev/null +++ b/lib/api/project_export.rb @@ -0,0 +1,41 @@ +module API + class ProjectExport < Grape::API + before do + not_found! unless Gitlab::CurrentSettings.project_export_enabled? + authorize_admin_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get export status' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectExportStatus + end + get ':id/export' do + present user_project, with: Entities::ProjectExportStatus + end + + desc 'Download export' do + detail 'This feature was introduced in GitLab 10.6.' + end + get ':id/export/download' do + path = user_project.export_project_path + + render_api_error!('404 Not found or has expired', 404) unless path + + present_file!(path, File.basename(path), 'application/gzip') + end + + desc 'Start export' do + detail 'This feature was introduced in GitLab 10.6.' + end + post ':id/export' do + user_project.add_export_job(current_user: current_user) + + accepted! + end + end + end +end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 86066e2b58f..f82241058e5 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -10,6 +10,7 @@ module API requires :url, type: String, desc: "The URL to send the request to" optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 5469cba69a6..7e6c33ec33d 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -16,7 +16,8 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) end post '/' do - attributes = attributes_for_keys [:description, :locked, :run_untagged, :tag_list] + attributes = attributes_for_keys([:description, :locked, :run_untagged, :tag_list]) + .merge(get_runner_details_from_request) runner = if runner_registration_token_valid? @@ -30,7 +31,6 @@ module API return forbidden! unless runner if runner.id - runner.update(get_runner_version_from_params) present runner, with: Entities::RunnerRegistrationDetails else not_found! @@ -204,6 +204,7 @@ module API optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) end @@ -224,7 +225,7 @@ module API expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata job.artifacts_expire_in = expire_in diff --git a/lib/api/services.rb b/lib/api/services.rb index 51e33e2c686..6c97659166d 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,6 +1,7 @@ +# frozen_string_literal: true module API class Services < Grape::API - chat_notification_settings = [ + CHAT_NOTIFICATION_SETTINGS = [ { required: true, name: :webhook, @@ -19,9 +20,9 @@ module API type: String, desc: 'The default chat channel' } - ] + ].freeze - chat_notification_flags = [ + CHAT_NOTIFICATION_FLAGS = [ { required: false, name: :notify_only_broken_pipelines, @@ -34,9 +35,9 @@ module API type: Boolean, desc: 'Send notifications only for the default branch' } - ] + ].freeze - chat_notification_channels = [ + CHAT_NOTIFICATION_CHANNELS = [ { required: false, name: :push_channel, @@ -85,9 +86,9 @@ module API type: String, desc: 'The name of the channel to receive wiki_page_events notifications' } - ] + ].freeze - chat_notification_events = [ + CHAT_NOTIFICATION_EVENTS = [ { required: false, name: :push_events, @@ -136,7 +137,7 @@ module API type: Boolean, desc: 'Enable notifications for wiki_page_events' } - ] + ].freeze services = { 'asana' => [ @@ -627,10 +628,10 @@ module API } ], 'slack' => [ - chat_notification_settings, - chat_notification_flags, - chat_notification_channels, - chat_notification_events + CHAT_NOTIFICATION_SETTINGS, + CHAT_NOTIFICATION_FLAGS, + CHAT_NOTIFICATION_CHANNELS, + CHAT_NOTIFICATION_EVENTS ].flatten, 'microsoft-teams' => [ { @@ -641,10 +642,10 @@ module API } ], 'mattermost' => [ - chat_notification_settings, - chat_notification_flags, - chat_notification_channels, - chat_notification_events + CHAT_NOTIFICATION_SETTINGS, + CHAT_NOTIFICATION_FLAGS, + CHAT_NOTIFICATION_CHANNELS, + CHAT_NOTIFICATION_EVENTS ].flatten, 'teamcity' => [ { @@ -724,7 +725,22 @@ module API ] end - trigger_services = { + SERVICES = services.freeze + SERVICE_CLASSES = service_classes.freeze + + SERVICE_CLASSES.each do |service| + event_names = service.try(:event_names) || next + event_names.each do |event_name| + SERVICES[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end + end + + TRIGGER_SERVICES = { 'mattermost-slash-commands' => [ { name: :token, @@ -756,22 +772,9 @@ module API end end - services.each do |service_slug, settings| + SERVICES.each do |service_slug, settings| desc "Set #{service_slug} service for project" params do - service_classes.each do |service| - event_names = service.try(:event_names) || next - event_names.each do |event_name| - services[service.to_param.tr("_", "-")] << { - required: false, - name: event_name.to_sym, - type: String, - desc: ServicesHelper.service_event_description(event_name) - } - end - end - services.freeze - settings.each do |setting| if setting[:required] requires setting[:name], type: setting[:type], desc: setting[:desc] @@ -794,7 +797,7 @@ module API desc "Delete a service for project" params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' end delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) @@ -814,7 +817,7 @@ module API success Entities::ProjectService end params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) @@ -822,7 +825,7 @@ module API end end - trigger_services.each do |service_slug, settings| + TRIGGER_SERVICES.each do |service_slug, settings| helpers do def slash_command_service(project, service_slug, params) project.services.active.where(template: false).find do |service| diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 2ccbb9da1c5..68b4d7c3982 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -252,8 +252,9 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :pipeline_events + expose :push_events, :issues_events, :confidential_issues_events + expose :merge_requests_events, :tag_push_events, :note_events + expose :pipeline_events expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| @@ -262,8 +263,9 @@ module API end class ProjectHook < ::API::Entities::Hook - expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :pipeline_events, :wiki_page_events + expose :project_id, :issues_events, :confidential_issues_events + expose :merge_requests_events, :note_events, :pipeline_events + expose :wiki_page_events expose :job_events, as: :build_events end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index d7bde8ceb89..88dd598f1e9 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -124,7 +124,7 @@ module API status(200 ) { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(source, current_user, declared_params).execute + ::Members::DestroyService.new(current_user).execute(member) present member, with: ::API::Entities::Member end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb index 51014591a93..631944150c7 100644 --- a/lib/api/v3/project_hooks.rb +++ b/lib/api/v3/project_hooks.rb @@ -11,6 +11,7 @@ module API requires :url, type: String, desc: "The URL to send the request to" optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index e7e6a90b5fd..c9e3f8ce42b 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -174,7 +174,9 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, parent, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, + link_content: !!link_content, + link_reference: link_reference) url = if matches.names.include?("url") && matches[:url] @@ -194,12 +196,13 @@ module Banzai end end - def data_attributes_for(text, project, object, link: false) + def data_attributes_for(text, project, object, link_content: false, link_reference: false) data_attribute( - original: text, - link: link, - project: project.id, - object_sym => object.id + original: text, + link: link_content, + link_reference: link_reference, + project: project.id, + object_sym => object.id ) end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index b8d2673c1a6..75b64ae9af2 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -25,8 +25,8 @@ module Banzai # period or comma for punctuation without those characters being included # in the generated link. # - # Rubular: http://rubular.com/r/cxjPyZc7Sb - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://\S+)(?<!,|\.)} + # Rubular: http://rubular.com/r/JzPhi6DCZp + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -35,53 +35,19 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') - and not(starts-with(., 'http')) - and not(starts-with(., 'ftp')) ]).freeze + PUNCTUATION_PAIRS = { + "'" => "'", + '"' => '"', + ')' => '(', + ']' => '[', + '}' => '{' + }.freeze + def call return doc if context[:autolink] == false - rinku_parse - text_parse - end - - private - - # Run the text through Rinku as a first pass - # - # This will quickly autolink http(s) and ftp links. - # - # `@doc` will be re-parsed with the HTML String from Rinku. - def rinku_parse - # Convert the options from a Hash to a String that Rinku expects - options = tag_options(link_options) - - # NOTE: We don't parse email links because it will erroneously match - # external Commit and CommitRange references. - # - # The final argument tells Rinku to link short URLs that don't include a - # period (e.g., http://localhost:3000/) - rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) - - return if rinku == html - - # Rinku returns a String, so parse it back to a Nokogiri::XML::Document - # for further processing. - @doc = parse_html(rinku) - end - - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - - # Autolinks any text matching LINK_PATTERN that Rinku didn't already - # replace - def text_parse doc.xpath(TEXT_QUERY).each do |node| content = node.to_html @@ -97,6 +63,16 @@ module Banzai doc end + private + + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + def autolink_match(match) # start by stripping out dangerous links begin @@ -112,12 +88,30 @@ module Banzai match.gsub!(/((?:&[\w#]+;)+)\z/, '') dropped = ($1 || '').html_safe + # To match the behaviour of Rinku, if the matched link ends with a + # closing part of a matched pair of punctuation, we remove that trailing + # character unless there are an equal number of closing and opening + # characters in the link. + if match.end_with?(*PUNCTUATION_PAIRS.keys) + close_character = match[-1] + close_count = match.count(close_character) + open_character = PUNCTUATION_PAIRS[close_character] + open_count = match.count(open_character) + + if open_count != close_count || open_character == close_character + dropped += close_character + match = match[0..-2] + end + end + options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + content_tag(:a, match.html_safe, options) + dropped end def autolink_filter(text) - text.gsub(LINK_PATTERN) { |match| autolink_match(match) } + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + autolink_match(link) + end end def link_options diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index eedb95197aa..43bf4fc6565 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -18,7 +18,8 @@ module Banzai def find_object(project, id) if project && project.valid_repo? - project.commit(id) + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 + Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } end end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb new file mode 100644 index 00000000000..bc9597df894 --- /dev/null +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -0,0 +1,45 @@ +# `CommonMark` markdown engine for GitLab's Banzai markdown filter. +# This module is used in Banzai::Filter::MarkdownFilter. +# Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser) +# including GitHub's GFM extensions. +# Homepage: https://github.com/gjtorikian/commonmarker + +module Banzai + module Filter + module MarkdownEngines + class CommonMark + EXTENSIONS = [ + :autolink, # provides support for automatically converting URLs to anchor tags. + :strikethrough, # provides support for strikethroughs. + :table, # provides support for tables. + :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension- + ].freeze + + PARSE_OPTIONS = [ + :FOOTNOTES, # parse footnotes. + :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does). + :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. + ].freeze + + # The `:GITHUB_PRE_LANG` option is not used intentionally because + # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>` + # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`. + # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below + # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`. + RENDER_OPTIONS = [ + :DEFAULT # default rendering system. Nothing special. + ].freeze + + def initialize + @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS) + end + + def render(text) + doc = CommonMarker.render_doc(text, PARSE_OPTIONS, EXTENSIONS) + + @renderer.render(doc) + end + end + end + end +end diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb new file mode 100644 index 00000000000..ac99941fefa --- /dev/null +++ b/lib/banzai/filter/markdown_engines/redcarpet.rb @@ -0,0 +1,32 @@ +# `Redcarpet` markdown engine for GitLab's Banzai markdown filter. +# This module is used in Banzai::Filter::MarkdownFilter. +# Used gem is `redcarpet` which is a ruby library for markdown processing. +# Homepage: https://github.com/vmg/redcarpet + +module Banzai + module Filter + module MarkdownEngines + class Redcarpet + OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + + def initialize + html_renderer = Banzai::Renderer::Redcarpet::HTML.new + @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS) + end + + def render(text) + @renderer.render(text) + end + end + end + end +end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 9cac303e645..c1e2b680240 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,34 +1,31 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - REDCARPET_OPTIONS = { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - def initialize(text, context = nil, result = nil) - super text, context, result - @text = @text.delete "\r" + super(text, context, result) + + @renderer = renderer(context[:markdown_engine]).new + @text = @text.delete("\r") end def call - html = self.class.renderer.render(@text) - html.rstrip! - html + @renderer.render(@text).rstrip + end + + private + + DEFAULT_ENGINE = :redcarpet + + def engine(engine_from_context) + engine_from_context ||= DEFAULT_ENGINE + + engine_from_context.to_s.classify end - def self.renderer - Thread.current[:banzai_markdown_renderer] ||= begin - renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) - end + def renderer(engine_from_context) + "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize + rescue NameError + raise NameError, "`#{engine_from_context}` is unknown markdown engine" end end end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 9bdedeb6615..262458a872a 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -84,7 +84,7 @@ module Banzai relative_url_root, project.full_path, uri_type(file_path), - Addressable::URI.escape(ref), + Addressable::URI.escape(ref).gsub('#', '%23'), Addressable::URI.escape(file_path) ].compact.join('/').squeeze('/').chomp('/') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 0ac7e231b5b..6dbf0d68fe8 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,3 +1,4 @@ +require 'rouge/plugins/common_mark' require 'rouge/plugins/redcarpet' module Banzai diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 827df7c08ae..fd457bebf03 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -42,16 +42,33 @@ module Banzai next if visible.include?(node) doc_data[:visible_reference_count] -= 1 - # The reference should be replaced by the original link's content, - # which is not always the same as the rendered one. - content = node.attr('data-original') || node.inner_html - node.replace(content) + redacted_content = redacted_node_content(node) + node.replace(redacted_content) end end metadata end + # Return redacted content of given node as either the original link (<a> tag), + # the original content (text), or the inner HTML of the node. + # + def redacted_node_content(node) + original_content = node.attr('data-original') + link_reference = node.attr('data-link-reference') + + # Build the raw <a> tag just with a link as href and content if + # it's originally a link pattern. We shouldn't return a plain text href. + original_link = + if link_reference == 'true' && href = original_content + %(<a href="#{href}">#{href}</a>) + end + + # The reference should be replaced by the original link's content, + # which is not always the same as the rendered one. + original_link || original_content || node.inner_html + end + def redact_cross_project_references(documents) extractor = Banzai::IssuableExtractor.new(project, user) issuables = extractor.extract(documents) diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb new file mode 100644 index 00000000000..c7a54629f31 --- /dev/null +++ b/lib/banzai/renderer/common_mark/html.rb @@ -0,0 +1,21 @@ +module Banzai + module Renderer + module CommonMark + class HTML < CommonMarker::HtmlRenderer + def code_block(node) + block do + code = node.string_content + lang = node.fence_info + lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' + result = + "<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + + out(result) + end + end + end + end + end +end diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb deleted file mode 100644 index 252caa35947..00000000000 --- a/lib/banzai/renderer/html.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Banzai - module Renderer - class HTML < Redcarpet::Render::HTML - def block_code(code, lang) - lang_attr = lang ? %Q{ lang="#{lang}"} : '' - - "\n<pre>" \ - "<code#{lang_attr}>#{html_escape(code)}</code>" \ - "</pre>" - end - end - end -end diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb new file mode 100644 index 00000000000..94df5d8b1e1 --- /dev/null +++ b/lib/banzai/renderer/redcarpet/html.rb @@ -0,0 +1,15 @@ +module Banzai + module Renderer + module Redcarpet + class HTML < ::Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + end + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index b9279c33f5b..ba5a9e2f04c 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -57,7 +57,7 @@ module Bitbucket end def provider - Gitlab::OAuth::Provider.config_for('bitbucket') + Gitlab::Auth::OAuth::Provider.config_for('bitbucket') end def options diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index fd2ac2db0a9..87649c50424 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,9 +1,11 @@ -class GroupUrlConstrainer - def matches?(request) - full_path = request.params[:group_id] || request.params[:id] +module Constraints + class GroupUrlConstrainer + def matches?(request) + full_path = request.params[:group_id] || request.params[:id] - return false unless NamespacePathValidator.valid_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) - Group.find_by_full_path(full_path, follow_redirects: request.get?).present? + Group.find_by_full_path(full_path, follow_redirects: request.get?).present? + end end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index e90ecb5ec69..32aea98f0f7 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -1,13 +1,15 @@ -class ProjectUrlConstrainer - def matches?(request) - namespace_path = request.params[:namespace_id] - project_path = request.params[:project_id] || request.params[:id] - full_path = [namespace_path, project_path].join('/') +module Constraints + class ProjectUrlConstrainer + def matches?(request) + namespace_path = request.params[:namespace_id] + project_path = request.params[:project_id] || request.params[:id] + full_path = [namespace_path, project_path].join('/') - return false unless ProjectPathValidator.valid_path?(full_path) + return false unless ProjectPathValidator.valid_path?(full_path) - # We intentionally allow SELECT(*) here so result of this query can be used - # as cache for further Project.find_by_full_path calls within request - Project.find_by_full_path(full_path, follow_redirects: request.get?).present? + # We intentionally allow SELECT(*) here so result of this query can be used + # as cache for further Project.find_by_full_path calls within request + Project.find_by_full_path(full_path, follow_redirects: request.get?).present? + end end end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 3b3ed1c6ddb..8afa04d29a4 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,9 +1,11 @@ -class UserUrlConstrainer - def matches?(request) - full_path = request.params[:username] +module Constraints + class UserUrlConstrainer + def matches?(request) + full_path = request.params[:username] - return false unless NamespacePathValidator.valid_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) - User.find_by_full_path(full_path, follow_redirects: request.get?).present? + User.find_by_full_path(full_path, follow_redirects: request.get?).present? + end end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index b1949d693ad..1dd2855063d 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -1,6 +1,8 @@ require_dependency 'declarative_policy/cache' require_dependency 'declarative_policy/condition' -require_dependency 'declarative_policy/dsl' +require_dependency 'declarative_policy/delegate_dsl' +require_dependency 'declarative_policy/policy_dsl' +require_dependency 'declarative_policy/rule_dsl' require_dependency 'declarative_policy/preferred_scope' require_dependency 'declarative_policy/rule' require_dependency 'declarative_policy/runner' diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb new file mode 100644 index 00000000000..f544dffe888 --- /dev/null +++ b/lib/declarative_policy/delegate_dsl.rb @@ -0,0 +1,16 @@ +module DeclarativePolicy + # Used when the name of a delegate is mentioned in + # the rule DSL. + class DelegateDsl + def initialize(rule_dsl, delegate_name) + @rule_dsl = rule_dsl + @delegate_name = delegate_name + end + + def method_missing(m, *a, &b) + return super unless a.empty? && !block_given? + + @rule_dsl.delegate(@delegate_name, m) + end + end +end diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb deleted file mode 100644 index 6ba1e7a3c5c..00000000000 --- a/lib/declarative_policy/dsl.rb +++ /dev/null @@ -1,103 +0,0 @@ -module DeclarativePolicy - # The DSL evaluation context inside rule { ... } blocks. - # Responsible for creating and combining Rule objects. - # - # See Base.rule - class RuleDsl - def initialize(context_class) - @context_class = context_class - end - - def can?(ability) - Rule::Ability.new(ability) - end - - def all?(*rules) - Rule::And.make(rules) - end - - def any?(*rules) - Rule::Or.make(rules) - end - - def none?(*rules) - ~Rule::Or.new(rules) - end - - def cond(condition) - Rule::Condition.new(condition) - end - - def delegate(delegate_name, condition) - Rule::DelegatedCondition.new(delegate_name, condition) - end - - def method_missing(m, *a, &b) - return super unless a.size == 0 && !block_given? - - if @context_class.delegations.key?(m) - DelegateDsl.new(self, m) - else - cond(m.to_sym) - end - end - end - - # Used when the name of a delegate is mentioned in - # the rule DSL. - class DelegateDsl - def initialize(rule_dsl, delegate_name) - @rule_dsl = rule_dsl - @delegate_name = delegate_name - end - - def method_missing(m, *a, &b) - return super unless a.size == 0 && !block_given? - - @rule_dsl.delegate(@delegate_name, m) - end - end - - # The return value of a rule { ... } declaration. - # Can call back to register rules with the containing - # Policy class (context_class here). See Base.rule - # - # Note that the #policy method just performs an #instance_eval, - # which is useful for multiple #enable or #prevent callse. - # - # Also provides a #method_missing proxy to the context - # class's class methods, so that helper methods can be - # defined and used in a #policy { ... } block. - class PolicyDsl - def initialize(context_class, rule) - @context_class = context_class - @rule = rule - end - - def policy(&b) - instance_eval(&b) - end - - def enable(*abilities) - @context_class.enable_when(abilities, @rule) - end - - def prevent(*abilities) - @context_class.prevent_when(abilities, @rule) - end - - def prevent_all - @context_class.prevent_all_when(@rule) - end - - def method_missing(m, *a, &b) - return super unless @context_class.respond_to?(m) - - @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(m) - @context_class.respond_to?(m) || super - end - end -end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb new file mode 100644 index 00000000000..f11b6e9f730 --- /dev/null +++ b/lib/declarative_policy/policy_dsl.rb @@ -0,0 +1,44 @@ +module DeclarativePolicy + # The return value of a rule { ... } declaration. + # Can call back to register rules with the containing + # Policy class (context_class here). See Base.rule + # + # Note that the #policy method just performs an #instance_eval, + # which is useful for multiple #enable or #prevent callse. + # + # Also provides a #method_missing proxy to the context + # class's class methods, so that helper methods can be + # defined and used in a #policy { ... } block. + class PolicyDsl + def initialize(context_class, rule) + @context_class = context_class + @rule = rule + end + + def policy(&b) + instance_eval(&b) + end + + def enable(*abilities) + @context_class.enable_when(abilities, @rule) + end + + def prevent(*abilities) + @context_class.prevent_when(abilities, @rule) + end + + def prevent_all + @context_class.prevent_all_when(@rule) + end + + def method_missing(m, *a, &b) + return super unless @context_class.respond_to?(m) + + @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(m) + @context_class.respond_to?(m) || super + end + end +end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index b0754098149..5c214408dd0 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,4 +1,4 @@ -module DeclarativePolicy +module DeclarativePolicy # rubocop:disable Naming/FileName PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" class << self diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb new file mode 100644 index 00000000000..e948b7f2de1 --- /dev/null +++ b/lib/declarative_policy/rule_dsl.rb @@ -0,0 +1,45 @@ +module DeclarativePolicy + # The DSL evaluation context inside rule { ... } blocks. + # Responsible for creating and combining Rule objects. + # + # See Base.rule + class RuleDsl + def initialize(context_class) + @context_class = context_class + end + + def can?(ability) + Rule::Ability.new(ability) + end + + def all?(*rules) + Rule::And.make(rules) + end + + def any?(*rules) + Rule::Or.make(rules) + end + + def none?(*rules) + ~Rule::Or.new(rules) + end + + def cond(condition) + Rule::Condition.new(condition) + end + + def delegate(delegate_name, condition) + Rule::DelegatedCondition.new(delegate_name, condition) + end + + def method_missing(m, *a, &b) + return super unless a.empty? && !block_given? + + if @context_class.delegations.key?(m) + DelegateDsl.new(self, m) + else + cond(m.to_sym) + end + end + end +end diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb index 7cb4bccb23c..91175b49c79 100644 --- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -3,7 +3,7 @@ require 'rails/generators' module Rails class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase def create_migration_file - timestamp = Time.now.strftime('%Y%m%d%H%I%S') + timestamp = Time.now.strftime('%Y%m%d%H%M%S') template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 11f7c8b9510..aa9fd36d9ff 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -2,6 +2,7 @@ require_dependency 'gitlab/git' module Gitlab COM_URL = 'https://gitlab.com'.freeze + APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} def self.com? # Check `staging?` as well to keep parity with gitlab.com diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 05932378173..f5ccf952cf9 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -40,8 +40,8 @@ module Gitlab end def find_with_user_password(login, password) - # Avoid resource intensive login checks if password is not provided - return unless password.present? + # Avoid resource intensive checks if login credentials are not provided + return unless login.present? && password.present? # Nothing to do here if internal auth is disabled and LDAP is # not configured @@ -50,14 +50,26 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - Gitlab::LDAP::Authentication.login(login, password) - elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user if user.active? && user.valid_password?(password) + return if user && !user.active? + + authenticators = [] + + if user + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, 'database') + + # Add authenticators for all identities if user is not nil + user&.identities&.each do |identity| + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, identity.provider) + end + else + # If no user is provided, try LDAP. + # LDAP users are only authenticated via LDAP + authenticators << Gitlab::Auth::LDAP::Authentication end + + authenticators.compact! + + user if authenticators.find { |auth| auth.login(login, password) } end end @@ -85,7 +97,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb new file mode 100644 index 00000000000..260a77058a4 --- /dev/null +++ b/lib/gitlab/auth/database/authentication.rb @@ -0,0 +1,16 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module Database + class Authentication < Gitlab::Auth::OAuth::Authentication + def login(login, password) + return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + + user&.valid_password?(password) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb new file mode 100644 index 00000000000..77c0ddc2d48 --- /dev/null +++ b/lib/gitlab/auth/ldap/access.rb @@ -0,0 +1,89 @@ +# LDAP authorization model +# +# * Check if we are allowed access (not blocked) +# +module Gitlab + module Auth + module LDAP + class Access + attr_reader :provider, :user + + def self.open(user, &block) + Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| + block.call(self.new(user, adapter)) + end + end + + def self.allowed?(user) + self.open(user) do |access| + if access.allowed? + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute + + true + else + false + end + end + end + + def initialize(user, adapter = nil) + @adapter = adapter + @user = user + @provider = user.ldap_identity.provider + end + + def allowed? + if ldap_user + unless ldap_config.active_directory + unblock_user(user, 'is available again') if user.ldap_blocked? + return true + end + + # Block user in GitLab if he/she was blocked in AD + if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) + block_user(user, 'is disabled in Active Directory') + false + else + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? + true + end + else + # Block the user if they no longer exist in LDAP/AD + block_user(user, 'does not exist anymore') + false + end + end + + def adapter + @adapter ||= Gitlab::Auth::LDAP::Adapter.new(provider) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def ldap_user + @ldap_user ||= Gitlab::Auth::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb new file mode 100644 index 00000000000..caf2d18c668 --- /dev/null +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -0,0 +1,110 @@ +module Gitlab + module Auth + module LDAP + class Adapter + attr_reader :provider, :ldap + + def self.open(provider, &block) + Net::LDAP.open(config(provider).adapter_options) do |ldap| + block.call(self.new(provider, ldap)) + end + end + + def self.config(provider) + Gitlab::Auth::LDAP::Config.new(provider) + end + + def initialize(provider, ldap = nil) + @provider = provider + @ldap = ldap || Net::LDAP.new(config.adapter_options) + end + + def config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) + + entries = ldap_search(options).select do |entry| + entry.respond_to? config.uid + end + + entries.map do |entry| + Gitlab::Auth::LDAP::Person.new(entry, provider) + end + end + + def user(*args) + users(*args).first + end + + def dn_matches_filter?(dn, filter) + ldap_search(base: dn, + filter: filter, + scope: Net::LDAP::SearchScope_BaseObject, + attributes: %w{dn}).any? + end + + def ldap_search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) + + if results.nil? + response = ldap.get_operation_result + + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end + + [] + else + results + end + end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] + end + + private + + def user_options(fields, value, limit) + options = { + attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), + base: config.base + } + + options[:size] = limit if limit + + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + else + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) + end + + options.merge(filter: user_filter(filter)) + end + + def user_filter(filter = nil) + user_filter = config.constructed_user_filter if config.user_filter.present? + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb new file mode 100644 index 00000000000..ac5c14d374d --- /dev/null +++ b/lib/gitlab/auth/ldap/auth_hash.rb @@ -0,0 +1,48 @@ +# Class to parse and transform the info provided by omniauth +# +module Gitlab + module Auth + module LDAP + class AuthHash < Gitlab::Auth::OAuth::AuthHash + def uid + @uid ||= Gitlab::Auth::LDAP::Person.normalize_dn(super) + end + + def username + super.tap do |username| + username.downcase! if ldap_config.lowercase_usernames + end + end + + private + + def get_info(key) + attributes = ldap_config.attributes[key.to_s] + return super unless attributes + + attributes = Array(attributes) + + value = nil + attributes.each do |attribute| + value = get_raw(attribute) + value = value.first if value + break if value.present? + end + + return super unless value + + Gitlab::Utils.force_utf8(value) + value + end + + def get_raw(key) + auth_hash.extra[:raw_info][key] if auth_hash.extra + end + + def ldap_config + @ldap_config ||= Gitlab::Auth::LDAP::Config.new(self.provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb new file mode 100644 index 00000000000..e70c3ab6b46 --- /dev/null +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -0,0 +1,68 @@ +# These calls help to authenticate to LDAP by providing username and password +# +# Since multiple LDAP servers are supported, it will loop through all of them +# until a valid bind is found +# + +module Gitlab + module Auth + module LDAP + class Authentication < Gitlab::Auth::OAuth::Authentication + def self.login(login, password) + return unless Gitlab::Auth::LDAP::Config.enabled? + return unless login.present? && password.present? + + auth = nil + # loop through providers until valid bind + providers.find do |provider| + auth = new(provider) + auth.login(login, password) # true will exit the loop + end + + # If (login, password) was invalid for all providers, the value of auth is now the last + # Gitlab::Auth::LDAP::Authentication instance we tried. + auth.user + end + + def self.providers + Gitlab::Auth::LDAP::Config.providers + end + + attr_accessor :ldap_user + + def login(login, password) + @ldap_user = adapter.bind_as( + filter: user_filter(login), + size: 1, + password: password + ) + end + + def adapter + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) + end + + def config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def user_filter(login) + filter = Net::LDAP::Filter.equals(config.uid, login) + + # Apply LDAP user filter if present + if config.user_filter.present? + filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) + end + + filter + end + + def user + return unless ldap_user + + Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb new file mode 100644 index 00000000000..77185f52ced --- /dev/null +++ b/lib/gitlab/auth/ldap/config.rb @@ -0,0 +1,237 @@ +# Load a specific server configuration +module Gitlab + module Auth + module LDAP + class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + + attr_accessor :provider, :options + + def self.enabled? + Gitlab.config.ldap.enabled + end + + def self.servers + Gitlab.config.ldap['servers']&.values || [] + end + + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + + def self.providers + servers.map { |server| server['provider_name'] } + end + + def self.valid_provider?(provider) + providers.include?(provider) + end + + def self.invalid_provider(provider) + raise "Unknown provider (#{provider}). Available providers: #{providers}" + end + + def initialize(provider) + if self.class.valid_provider?(provider) + @provider = provider + else + self.class.invalid_provider(provider) + end + + @options = config_for(@provider) # Use @provider, not provider + end + + def enabled? + base_config.enabled + end + + def adapter_options + opts = base_options.merge( + encryption: encryption_options + ) + + opts.merge!(auth_options) if has_auth? + + opts + end + + def omniauth_options + opts = base_options.merge( + base: base, + encryption: options['encryption'], + filter: omniauth_user_filter, + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] + ) + + if has_auth? + opts.merge!( + bind_dn: options['bind_dn'], + password: options['password'] + ) + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts + end + + def base + options['base'] + end + + def uid + options['uid'] + end + + def sync_ssh_keys? + sync_ssh_keys.present? + end + + # The LDAP attribute in which the ssh keys are stored + def sync_ssh_keys + options['sync_ssh_keys'] + end + + def user_filter + options['user_filter'] + end + + def constructed_user_filter + @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) + end + + def group_base + options['group_base'] + end + + def admin_group + options['admin_group'] + end + + def active_directory + options['active_directory'] + end + + def block_auto_created_users + options['block_auto_created_users'] + end + + def attributes + default_attributes.merge(options['attributes']) + end + + def timeout + options['timeout'].to_i + end + + def has_auth? + options['password'] || options['bind_dn'] + end + + def allow_username_or_email_login + options['allow_username_or_email_login'] + end + + def lowercase_usernames + options['lowercase_usernames'] + end + + def name_proc + if allow_username_or_email_login + proc { |name| name.gsub(/@.*\z/, '') } + else + proc { |name| name } + end + end + + def default_attributes + { + 'username' => %w(uid sAMAccountName userid), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + + protected + + def base_options + { + host: options['host'], + port: options['port'] + } + end + + def base_config + Gitlab.config.ldap + end + + def config_for(provider) + base_config.servers.values.find { |server| server['provider_name'] == provider } + end + + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts + end + + def auth_options + { + auth: { + method: :simple, + username: options['bind_dn'], + password: options['password'] + } + } + end + + def omniauth_user_filter + uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') + + if user_filter.present? + Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s + else + uid_filter.to_s + end + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb new file mode 100644 index 00000000000..1fa5338f5a6 --- /dev/null +++ b/lib/gitlab/auth/ldap/dn.rb @@ -0,0 +1,303 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module Auth + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb new file mode 100644 index 00000000000..8dfae3ee541 --- /dev/null +++ b/lib/gitlab/auth/ldap/person.rb @@ -0,0 +1,122 @@ +module Gitlab + module Auth + module LDAP + class Person + # Active Directory-specific LDAP filter that checks if bit 2 of the + # userAccountControl attribute is set. + # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ + AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + + InvalidEntryError = Class.new(StandardError) + + attr_accessor :entry, :provider + + def self.find_by_uid(uid, adapter) + uid = Net::LDAP::Filter.escape(uid) + adapter.user(adapter.config.uid, uid) + end + + def self.find_by_dn(dn, adapter) + adapter.user('dn', dn) + end + + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + + def self.disabled_via_active_directory?(dn, adapter) + adapter.dn_matches_filter?(dn, AD_USER_DISABLED) + end + + def self.ldap_attributes(config) + [ + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq + end + + def self.normalize_dn(dn) + ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::Auth::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + + def initialize(entry, provider) + Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } + @entry = entry + @provider = provider + end + + def name + attribute_value(:name).first + end + + def uid + entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend + end + + def username + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first.tap do |username| + username.downcase! if config.lowercase_usernames + end + end + + def email + attribute_value(:email) + end + + def dn + self.class.normalize_dn(entry.dn) + end + + private + + def entry + @entry + end + + def config + @config ||= Gitlab::Auth::LDAP::Config.new(provider) + end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_s]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb new file mode 100644 index 00000000000..068212d9a21 --- /dev/null +++ b/lib/gitlab/auth/ldap/user.rb @@ -0,0 +1,54 @@ +# LDAP extension for User model +# +# * Find or create user from omniauth.auth data +# * Links LDAP account with existing user +# * Auth LDAP user with login and password +# +module Gitlab + module Auth + module LDAP + class User < Gitlab::Auth::OAuth::User + class << self + def find_by_uid_and_provider(uid, provider) + identity = ::Identity.with_extern_uid(provider, uid).take + + identity && identity.user + end + end + + def save + super('LDAP') + end + + # instance methods + def find_user + find_by_uid_and_provider || find_by_email || build_new_user + end + + def find_by_uid_and_provider + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) + end + + def changed? + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + + def block_after_signup? + ldap_config.block_auto_created_users + end + + def allowed? + Gitlab::Auth::LDAP::Access.allowed?(gl_user) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(auth_hash.provider) + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Auth::LDAP::AuthHash.new(auth_hash) + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb new file mode 100644 index 00000000000..ed8fba94305 --- /dev/null +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -0,0 +1,92 @@ +# Class to parse and transform the info provided by omniauth +# +module Gitlab + module Auth + module OAuth + class AuthHash + attr_reader :auth_hash + def initialize(auth_hash) + @auth_hash = auth_hash + end + + def uid + @uid ||= Gitlab::Utils.force_utf8(auth_hash.uid.to_s) + end + + def provider + @provider ||= auth_hash.provider.to_s + end + + def name + @name ||= get_info(:name) || "#{get_info(:first_name)} #{get_info(:last_name)}" + end + + def username + @username ||= username_and_email[:username].to_s + end + + def email + @email ||= username_and_email[:email].to_s + end + + def password + @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) + end + + def location + location = get_info(:address) + if location.is_a?(Hash) + [location.locality.presence, location.country.presence].compact.join(', ') + else + location + end + end + + def has_attribute?(attribute) + if attribute == :location + get_info(:address).present? + else + get_info(attribute).present? + end + end + + private + + def info + auth_hash.info + end + + def get_info(key) + value = info[key] + Gitlab::Utils.force_utf8(value) if value + value + end + + def username_and_email + @username_and_email ||= begin + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence + + username ||= generate_username(email) if email + email ||= generate_temporarily_email(username) if username + + { + username: username, + email: email + } + end + end + + # Get the first part of the email address (before @) + # In addtion in removes illegal characters + def generate_username(email) + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s + end + + def generate_temporarily_email(username) + "temp-email-for-oauth-#{username}@gitlab.localhost" + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb new file mode 100644 index 00000000000..ed03b9f8b40 --- /dev/null +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -0,0 +1,21 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module OAuth + class Authentication + attr_reader :provider, :user + + def initialize(provider, user = nil) + @provider = provider + @user = user + end + + def login(login, password) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb new file mode 100644 index 00000000000..5fb61ffe00d --- /dev/null +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -0,0 +1,73 @@ +module Gitlab + module Auth + module OAuth + class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + + def self.authentication(user, provider) + return unless user + return unless enabled?(provider) + + authenticator = + case provider + when /^ldap/ + Gitlab::Auth::LDAP::Authentication + when 'database' + Gitlab::Auth::Database::Authentication + end + + authenticator&.new(provider, user) + end + + def self.providers + Devise.omniauth_providers + end + + def self.enabled?(name) + return true if name == 'database' + + providers.include?(name.to_sym) + end + + def self.ldap_provider?(name) + name.to_s.start_with?('ldap') + end + + def self.sync_profile_from_provider?(provider) + return true if ldap_provider?(provider) + + providers = Gitlab.config.omniauth.sync_profile_from_provider + + if providers.is_a?(Array) + providers.include?(provider) + else + providers + end + end + + def self.config_for(name) + name = name.to_s + if ldap_provider?(name) + if Gitlab::Auth::LDAP::Config.valid_provider?(name) + Gitlab::Auth::LDAP::Config.new(name).options + else + nil + end + else + Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + end + end + + def self.label_for(name) + name = name.to_s + config = config_for(name) + (config && config['label']) || LABELS[name] || name.titleize + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb new file mode 100644 index 00000000000..8f2b4d58552 --- /dev/null +++ b/lib/gitlab/auth/o_auth/session.rb @@ -0,0 +1,21 @@ +# :nocov: +module Gitlab + module Auth + module OAuth + module Session + def self.create(provider, ticket) + Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) + end + + def self.destroy(provider, ticket) + Rails.cache.delete("gitlab:#{provider}:#{ticket}") + end + + def self.valid?(provider, ticket) + Rails.cache.read("gitlab:#{provider}:#{ticket}").present? + end + end + end + end +end +# :nocov: diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb new file mode 100644 index 00000000000..b6a96081278 --- /dev/null +++ b/lib/gitlab/auth/o_auth/user.rb @@ -0,0 +1,246 @@ +# OAuth extension for User model +# +# * Find GitLab user based on omniauth uid and provider +# * Create new user from omniauth data +# +module Gitlab + module Auth + module OAuth + class User + SignupDisabledError = Class.new(StandardError) + SigninDisabledForProviderError = Class.new(StandardError) + + attr_accessor :auth_hash, :gl_user + + def initialize(auth_hash) + self.auth_hash = auth_hash + update_profile + add_or_update_user_identities + end + + def persisted? + gl_user.try(:persisted?) + end + + def new? + !persisted? + end + + def valid? + gl_user.try(:valid?) + end + + def save(provider = 'OAuth') + raise SigninDisabledForProviderError if oauth_provider_disabled? + raise SignupDisabledError unless gl_user + + block_after_save = needs_blocking? + + Users::UpdateService.new(gl_user, user: gl_user).execute! + + gl_user.block if block_after_save + + log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + gl_user + rescue ActiveRecord::RecordInvalid => e + log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" + return self, e.record.errors + end + + def gl_user + return @gl_user if defined?(@gl_user) + + @gl_user = find_user + end + + def find_user + user = find_by_uid_and_provider + + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user&.new_record? + + user + end + + protected + + def add_or_update_user_identities + return unless gl_user + + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user + return unless ldap_person + + user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) + if user + log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." + return user + end + + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) + end + + def auto_link_ldap_user? + Gitlab.config.omniauth.auto_link_ldap_user + end + + def creating_linked_ldap_user? + auto_link_ldap_user? && ldap_person + end + + def ldap_person + return @ldap_person if defined?(@ldap_person) + + # Look for a corresponding person with same uid in any of the configured LDAP providers + Gitlab::Auth::LDAP::Config.providers.each do |provider| + adapter = Gitlab::Auth::LDAP::Adapter.new(provider) + @ldap_person = find_ldap_person(auth_hash, adapter) + break if @ldap_person + end + @ldap_person + end + + def find_ldap_person(auth_hash, adapter) + Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(ldap_person.provider) if ldap_person + end + + def needs_blocking? + new? && block_after_signup? + end + + def signup_enabled? + providers = Gitlab.config.omniauth.allow_single_sign_on + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end + end + + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + + def block_after_signup? + if creating_linked_ldap_user? + ldap_config.block_auto_created_users + else + Gitlab.config.omniauth.block_auto_created_users + end + end + + def auth_hash=(auth_hash) + @auth_hash = AuthHash.new(auth_hash) + end + + def find_by_uid_and_provider + identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take + identity&.user + end + + def build_new_user + user_params = user_attributes.merge(skip_confirmation: true) + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) + end + + def user_attributes + # Give preference to LDAP for sensitive information when creating a linked account + if creating_linked_ldap_user? + username = ldap_person.username.presence + email = ldap_person.email.first.presence + end + + username ||= auth_hash.username + email ||= auth_hash.email + + valid_username = ::Namespace.clean_path(username) + + uniquify = Uniquify.new + valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } + + name = auth_hash.name + name = valid_username if name.strip.empty? + + { + name: name, + username: valid_username, + email: email, + password: auth_hash.password, + password_confirmation: auth_hash.password, + password_automatically_set: true + } + end + + def sync_profile_from_provider? + Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) + end + + def update_profile + clear_user_synced_attributes_metadata + + return unless sync_profile_from_provider? || creating_linked_ldap_user? + + metadata = gl_user.build_user_synced_attributes_metadata + + if sync_profile_from_provider? + UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) + gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + metadata.set_attribute_synced(key, true) + else + metadata.set_attribute_synced(key, false) + end + end + + metadata.provider = auth_hash.provider + end + + if creating_linked_ldap_user? && gl_user.email == ldap_person.email.first + metadata.set_attribute_synced(:email, true) + metadata.provider = ldap_person.provider + end + end + + def clear_user_synced_attributes_metadata + gl_user&.user_synced_attributes_metadata&.destroy + end + + def log + Gitlab::AppLogger + end + + def oauth_provider_disabled? + Gitlab::CurrentSettings.current_application_settings + .disabled_oauth_sign_in_sources + .include?(auth_hash.provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 75451cf8aa9..00cdc94a9ef 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -1,4 +1,4 @@ -module Gitlab +module Gitlab # rubocop:disable Naming/FileName module Auth Result = Struct.new(:actor, :project, :type, :authentication_abilities) do def ci?(for_project) diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb new file mode 100644 index 00000000000..c345a7e3f6c --- /dev/null +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Auth + module Saml + class AuthHash < Gitlab::Auth::OAuth::AuthHash + def groups + Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb new file mode 100644 index 00000000000..2760b1a3247 --- /dev/null +++ b/lib/gitlab/auth/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Auth + module Saml + class Config + class << self + def options + Gitlab::Auth::OAuth::Provider.config_for('saml') + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb new file mode 100644 index 00000000000..d4024e9ec39 --- /dev/null +++ b/lib/gitlab/auth/saml/user.rb @@ -0,0 +1,52 @@ +# SAML extension for User model +# +# * Find GitLab user based on SAML uid and provider +# * Create new user from SAML data +# +module Gitlab + module Auth + module Saml + class User < Gitlab::Auth::OAuth::User + def save + super('SAML') + end + + def find_user + user = find_by_uid_and_provider + + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + if external_users_enabled? && user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + user.external = !(auth_hash.groups & Gitlab::Auth::Saml::Config.external_groups).empty? + end + + user + end + + def changed? + return true unless gl_user + + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + + protected + + def auto_link_saml_user? + Gitlab.config.omniauth.auto_link_saml_user + end + + def external_users_enabled? + !Gitlab::Auth::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Auth::Saml::AuthHash.new(auth_hash) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index 7bffffec94d..d5cf9e0d53a 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -19,7 +19,7 @@ module Gitlab WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id )'.squish - MergeRequestDiff.where(id: start_id..stop_id).update_all(update) + MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update) end end end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb new file mode 100644 index 00000000000..8fe4f1a2289 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class MigrateBuildStage + module Migratable + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + def ensure_stage!(attempts: 2) + find_stage || create_stage! + rescue ActiveRecord::RecordNotUnique + retry if (attempts -= 1) > 0 + raise + end + + def find_stage + Stage.find_by(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + + def create_stage! + Stage.create!(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + end + end + + def perform(start_id, stop_id) + stages = Migratable::Build.where('stage_id IS NULL') + .where('id BETWEEN ? AND ?', start_id, stop_id) + .map { |build| build.ensure_stage! } + .compact.map(&:id) + + MigrateBuildStageIdReference.new.perform(start_id, stop_id) + MigrateStageStatus.new.perform(stages.min, stages.max) + end + end + end +end diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index 85749366bfd..d9d3d2e667b 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -16,281 +16,283 @@ module Gitlab # And if the normalize behavior is changed in the future, it must be # accompanied by another migration. module Gitlab - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) + module Auth + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end end - end - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" end - end - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end end - end - str - end + str + end - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end - ## - # Return the DN as an escaped string. - def to_s - @dn - end + ## + # Return the DN as an escaped string. + def to_s + @dn + end - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end - private + private - def initialize_array(args) - buffer = StringIO.new + def initialize_array(args) + buffer = StringIO.new - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end end - end - @dn = buffer.string - end + @dn = buffer.string + end - def initialize_string(arg) - @dn = arg.to_s - end + def initialize_string(arg) + @dn = arg.to_s + end - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end end end end @@ -302,11 +304,11 @@ module Gitlab ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) ldap_identities.each do |identity| begin - identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + identity.extern_uid = Gitlab::Auth::LDAP::DN.new(identity.extern_uid).to_normalized_s unless identity.save Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." end - rescue Gitlab::LDAP::DN::FormatError => e + rescue Gitlab::Auth::LDAP::DN::FormatError => e Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d48ae17aeaf..bffbcb86137 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -135,7 +135,7 @@ module Gitlab if label.valid? @labels[label_params[:title]] = label else - raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.name_with_namespace}\"" + raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 3ce5f807989..51ba09aa129 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -47,7 +47,7 @@ module Gitlab protected def push_checks - if user_access.cannot_do_action?(:push_code) + unless can_push? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] end end @@ -183,6 +183,11 @@ module Gitlab def commits @commits ||= project.repository.new_commits(newrev) end + + def can_push? + user_access.can_do_action?(:push_code) || + user_access.can_push_to_branch?(branch_name) + end end end end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 525563a97f5..46ed330dbbf 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -68,10 +68,11 @@ module Gitlab class YearChart < Chart include MonthlyInterval + attr_reader :to, :from def initialize(*) - @to = Date.today.end_of_month - @from = @to.years_ago(1).beginning_of_month + @to = Date.today.end_of_month.end_of_day + @from = @to.years_ago(1).beginning_of_month.beginning_of_day @format = '%d %B %Y' super @@ -80,10 +81,11 @@ module Gitlab class MonthChart < Chart include DailyInterval + attr_reader :to, :from def initialize(*) - @to = Date.today - @from = @to - 30.days + @to = Date.today.end_of_day + @from = 1.month.ago.beginning_of_day @format = '%d %B' super @@ -92,10 +94,11 @@ module Gitlab class WeekChart < Chart include DailyInterval + attr_reader :to, :from def initialize(*) - @to = Date.today - @from = @to - 7.days + @to = Date.today.end_of_day + @from = 1.week.ago.beginning_of_day @format = '%d %B' super diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 7b19b10e05b..a1849b01c5d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,4 @@ -module Gitlab +module Gitlab # rubocop:disable Naming/FileName module Ci module Pipeline module Chain diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d19a2519803..d5e17a123df 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,27 +17,11 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") - ensure - if pipeline.builds.where(stage_id: nil).any? - invalid_builds_counter.increment(node: hostname) - end end def break? !pipeline.persisted? end - - private - - def invalid_builds_counter - @counter ||= Gitlab::Metrics - .counter(:gitlab_ci_invalid_builds_total, - 'Invalid builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb new file mode 100644 index 00000000000..047ab66e9b3 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Base + def evaluate(**variables) + raise NotImplementedError + end + + def self.build(token) + raise NotImplementedError + end + + def self.scan(scanner) + if scanner.scan(self::PATTERN) + Expression::Token.new(scanner.matched, self) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb new file mode 100644 index 00000000000..3a2f0c6924e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Equals < Lexeme::Operator + PATTERN = /==/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + @left.evaluate(variables) == @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb new file mode 100644 index 00000000000..a2778716924 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Null < Lexeme::Value + PATTERN = /null/.freeze + + def initialize(value = nil) + @value = nil + end + + def evaluate(variables = {}) + nil + end + + def self.build(_value) + self.new + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb new file mode 100644 index 00000000000..f640d0b5855 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Operator < Lexeme::Base + def self.type + :operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb new file mode 100644 index 00000000000..48bde213d44 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class String < Lexeme::Value + PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + + def initialize(value) + @value = value + end + + def evaluate(variables = {}) + @value.to_s + end + + def self.build(string) + new(string.match(PATTERN)[:string]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb new file mode 100644 index 00000000000..f2611d65faf --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Value < Lexeme::Base + def self.type + :value + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb new file mode 100644 index 00000000000..b781c15fd67 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Variable < Lexeme::Value + PATTERN = /\$(?<name>\w+)/.freeze + + def initialize(name) + @name = name + end + + def evaluate(variables = {}) + HashWithIndifferentAccess.new(variables).fetch(@name, nil) + end + + def self.build(string) + new(string.match(PATTERN)[:name]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb new file mode 100644 index 00000000000..e1c68b7c3c2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -0,0 +1,59 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Lexer + include ::Gitlab::Utils::StrongMemoize + + LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Null, + Expression::Lexeme::Equals + ].freeze + + SyntaxError = Class.new(Statement::StatementError) + + MAX_TOKENS = 100 + + def initialize(statement, max_tokens: MAX_TOKENS) + @scanner = StringScanner.new(statement) + @max_tokens = max_tokens + end + + def tokens + strong_memoize(:tokens) { tokenize } + end + + def lexemes + tokens.map(&:to_lexeme) + end + + private + + def tokenize + tokens = [] + + @max_tokens.times do + @scanner.skip(/\s+/) # ignore whitespace + + return tokens if @scanner.eos? + + lexeme = LEXEMES.find do |type| + type.scan(@scanner).tap do |token| + tokens.push(token) if token.present? + end + end + + unless lexeme.present? + raise Lexer::SyntaxError, 'Unknown lexeme found!' + end + end + + raise Lexer::SyntaxError, 'Too many tokens!' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb new file mode 100644 index 00000000000..90f94d0b763 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Parser + def initialize(tokens) + @tokens = tokens.to_enum + @nodes = [] + end + + ## + # This produces a reverse descent parse tree. + # + # It currently does not support precedence of operators. + # + def tree + while token = @tokens.next + case token.type + when :operator + token.build(@nodes.pop, tree).tap do |node| + @nodes.push(node) + end + when :value + token.build.tap do |leaf| + @nodes.push(leaf) + end + end + end + rescue StopIteration + @nodes.last || Lexeme::Null.new + end + + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb new file mode 100644 index 00000000000..4f0e101b730 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -0,0 +1,42 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Statement + StatementError = Class.new(StandardError) + + GRAMMAR = [ + %w[variable equals string], + %w[variable equals variable], + %w[variable equals null], + %w[string equals variable], + %w[null equals variable], + %w[variable] + ].freeze + + def initialize(statement, pipeline) + @lexer = Expression::Lexer.new(statement) + + @variables = pipeline.variables.map do |variable| + [variable.key, variable.value] + end + end + + def parse_tree + raise StatementError if @lexer.lexemes.empty? + + unless GRAMMAR.find { |syntax| syntax == @lexer.lexemes } + raise StatementError, 'Unknown pipeline expression!' + end + + Expression::Parser.new(@lexer.tokens).tree + end + + def evaluate + parse_tree.evaluate(@variables.to_h) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb new file mode 100644 index 00000000000..58211800b88 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -0,0 +1,28 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Token + attr_reader :value, :lexeme + + def initialize(value, lexeme) + @value = value + @lexeme = lexeme + end + + def build(*args) + @lexeme.build(@value, *args) + end + + def type + @lexeme.type + end + + def to_lexeme + @lexeme.name.demodulize.downcase + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f2e5124c8a8..cedf4171ab1 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,6 +1,8 @@ module Gitlab module Ci class Trace + ArchiveError = Class.new(StandardError) + attr_reader :job delegate :old_trace, to: :job @@ -93,8 +95,53 @@ module Gitlab job.erase_old_trace! end + def archive! + raise ArchiveError, 'Already archived' if trace_artifact + raise ArchiveError, 'Job is not finished yet' unless job.complete? + + if current_path + File.open(current_path) do |stream| + archive_stream!(stream) + FileUtils.rm(current_path) + end + elsif old_trace + StringIO.new(old_trace, 'rb').tap do |stream| + archive_stream!(stream) + job.erase_old_trace! + end + end + end + private + def archive_stream!(stream) + clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| + create_job_trace!(job, clone_path) + end + end + + def clone_file!(src_stream, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.touch(temp_path) + size = IO.copy_stream(src_stream, temp_path) + raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size + + yield(temp_path) + end + end + + def create_job_trace!(job, path) + File.open(path) do |stream| + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream, + file_sha256: Digest::SHA256.file(path).hexdigest) + end + end + def ensure_path return current_path if current_path diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb new file mode 100644 index 00000000000..0deca55fe8f --- /dev/null +++ b/lib/gitlab/ci/variables/collection.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + module Variables + class Collection + include Enumerable + + def initialize(variables = []) + @variables = [] + + variables.each { |variable| self.append(variable) } + end + + def append(resource) + tap { @variables.append(Collection::Item.fabricate(resource)) } + end + + def concat(resources) + tap { resources.each { |variable| self.append(variable) } } + end + + def each + @variables.each { |variable| yield variable } + end + + def +(other) + self.class.new.tap do |collection| + self.each { |variable| collection.append(variable) } + other.each { |variable| collection.append(variable) } + end + end + + def to_runner_variables + self.map(&:to_hash) + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb new file mode 100644 index 00000000000..939912981e6 --- /dev/null +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -0,0 +1,50 @@ +module Gitlab + module Ci + module Variables + class Collection + class Item + def initialize(**options) + @variable = { + key: options.fetch(:key), + value: options.fetch(:value), + public: options.fetch(:public, true), + file: options.fetch(:files, false) + } + end + + def [](key) + @variable.fetch(key) + end + + def ==(other) + to_hash == self.class.fabricate(other).to_hash + end + + ## + # If `file: true` has been provided we expose it, otherwise we + # don't expose `file` attribute at all (stems from what the runner + # expects). + # + def to_hash + @variable.reject do |hash_key, hash_value| + hash_key == :file && hash_value == false + end + end + + def self.fabricate(resource) + case resource + when Hash + self.new(resource) + when ::HasVariable + self.new(resource.to_runner_variable) + when self + resource.dup + else + raise ArgumentError, "Unknown `#{resource.class}` variable resource!" + end + end + end + end + end + end +end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 0a3ae2c3760..3ccfd9a739d 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,14 +1,18 @@ module Gitlab module Conflict class FileCollection + include Gitlab::RepositoryCacheAdapter + attr_reader :merge_request, :resolver def initialize(merge_request) our_commit = merge_request.source_branch_head.raw their_commit = merge_request.target_branch_head.raw - target_repo = merge_request.target_project.repository.raw + @target_repo = merge_request.target_project.repository @source_repo = merge_request.source_project.repository.raw - @resolver = Gitlab::Git::Conflict::Resolver.new(target_repo, our_commit.id, their_commit.id) + @our_commit_id = our_commit.id + @their_commit_id = their_commit.id + @resolver = Gitlab::Git::Conflict::Resolver.new(@target_repo.raw, @our_commit_id, @their_commit_id) @merge_request = merge_request end @@ -30,6 +34,17 @@ module Gitlab end end + def can_be_resolved_in_ui? + # Try to parse each conflict. If the MR's mergeable status hasn't been + # updated, ensure that we don't say there are conflicts to resolve + # when there are no conflict files. + files.each(&:lines) + files.any? + rescue Gitlab::Git::CommandError, Gitlab::Git::Conflict::Parser::UnresolvableError, Gitlab::Git::Conflict::Resolver::ConflictSideMissing + false + end + cache_method :can_be_resolved_in_ui? + def file_for_path(old_path, new_path) files.find { |file| file.their_path == old_path && file.our_path == new_path } end @@ -56,6 +71,19 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end + + private + + def cache + @cache ||= begin + # Use the commit ids as a namespace so if the MR branches get + # updated we instantiate the cache under a different namespace. That + # way don't have to worry about explicitly invalidating the cache + namespace = "#{@our_commit_id}:#{@their_commit_id}" + + Gitlab::RepositoryCache.new(@target_repo, extra_namespace: namespace) + end + end end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9576d5a3fd8..d7369060cc5 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -23,7 +23,7 @@ module Gitlab mr_events = event_counts(date_from, :merge_requests) .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") note_events = event_counts(date_from, :merge_requests) - .having(action: [Event::COMMENTED], target_type: "Note") + .having(action: [Event::COMMENTED]) union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 8b3bc3e440d..86d708be0d6 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -8,13 +8,14 @@ module Gitlab private def base_query - @base_query ||= stage_query + @base_query ||= stage_query(@project.id) # rubocop:disable Gitlab/ModuleWithInstanceVariables end - def stage_query + def stage_query(project_ids) query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .project(issue_table[:project_id].as("project_id")) + .where(issue_table[:project_id].in(project_ids)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index cac31ea8cff..038d5a19bc4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -21,17 +21,28 @@ module Gitlab end def median - cte_table = Arel::Table.new("cte_table_for_#{name}") + BatchLoader.for(@project.id).batch(key: name) do |project_ids, loader| + cte_table = Arel::Table.new("cte_table_for_#{name}") - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new( - cte_table, - subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new(cte_table, + subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) - median_datetime(cte_table, interval_query, name) + if project_ids.one? + loader.call(@project.id, median_datetime(cte_table, interval_query, name)) + else + begin + median_datetimes(cte_table, interval_query, name, :project_id)&.each do |project_id, median| + loader.call(project_id, median) + end + rescue NotSupportedError + {} + end + end + end end def name diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index 7a889b3877f..d0ca62e46e4 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -1,8 +1,8 @@ module Gitlab module CycleAnalytics module ProductionHelper - def stage_query - super + def stage_query(project_ids) + super(project_ids) .where(mr_metrics_table[:first_deployed_to_production_at] .gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index 2b5f72bef89..0e9d235ca79 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -25,11 +25,11 @@ module Gitlab _("Total test time for all commits/merges") end - def stage_query + def stage_query(project_ids) if @options[:branch] - super.where(build_table[:ref].eq(@options[:branch])) + super(project_ids).where(build_table[:ref].eq(@options[:branch])) else - super + super(project_ids) end end end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb new file mode 100644 index 00000000000..5122e3417ca --- /dev/null +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -0,0 +1,72 @@ +module Gitlab + module CycleAnalytics + class UsageData + PROJECTS_LIMIT = 10 + + attr_reader :projects, :options + + def initialize + @projects = Project.sorted_by_activity.limit(PROJECTS_LIMIT) + @options = { from: 7.days.ago } + end + + def to_json + total = 0 + + values = + medians_per_stage.each_with_object({}) do |(stage_name, medians), hsh| + calculations = stage_values(medians) + + total += calculations.values.compact.sum + hsh[stage_name] = calculations + end + + values[:total] = total + + { avg_cycle_analytics: values } + end + + private + + def medians_per_stage + projects.each_with_object({}) do |project, hsh| + ::CycleAnalytics.new(project, options).all_medians_per_stage.each do |stage_name, median| + hsh[stage_name] ||= [] + hsh[stage_name] << median + end + end + end + + def stage_values(medians) + medians = medians.map(&:presence).compact + average = calc_average(medians) + + { + average: average, + sd: standard_deviation(medians, average), + missing: projects.length - medians.length + } + end + + def calc_average(values) + return if values.empty? + + (values.sum / values.length).to_i + end + + def standard_deviation(values, average) + Math.sqrt(sample_variance(values, average)).to_i + end + + def sample_variance(values, average) + return 0 if values.length <= 1 + + sum = values.inject(0) do |acc, val| + acc + (val - average)**2 + end + + sum / (values.length - 1) + end + end + end +end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 8e74e18a311..2f1445a050a 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -31,7 +31,7 @@ module Gitlab # TODO: do we still need it? project_id: project.id, - project_name: project.name_with_namespace, + project_name: project.full_name, user: { id: user.try(:id), diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index e47fb85b5ee..1e283cc092b 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,6 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, + detailed_status: pipeline.detailed_status(nil).label, stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 059054ac9ff..74fed447289 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -2,18 +2,14 @@ module Gitlab module Database module Median + NotSupportedError = Class.new(StandardError) + def median_datetime(arel_table, query_so_far, column_sym) - median_queries = - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym) - elsif Gitlab::Database.mysql? - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - - results = Array.wrap(median_queries).map do |query| - ActiveRecord::Base.connection.execute(query) - end - extract_median(results).presence + extract_median(execute_queries(arel_table, query_so_far, column_sym)).presence + end + + def median_datetimes(arel_table, query_so_far, column_sym, partition_column) + extract_medians(execute_queries(arel_table, query_so_far, column_sym, partition_column)).presence end def extract_median(results) @@ -21,13 +17,21 @@ module Gitlab if Gitlab::Database.postgresql? result = result.first.presence - median = result['median'] if result - median.to_f if median + + result['median']&.to_f if result elsif Gitlab::Database.mysql? result.to_a.flatten.first end end + def extract_medians(results) + median_values = results.compact.first.values + + median_values.each_with_object({}) do |(id, median), hash| + hash[id.to_i] = median&.to_f + end + end + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) query = arel_table .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) @@ -53,7 +57,7 @@ module Gitlab ] end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows # of the CTE. For example, if we're looking to find the median of the `projects.star_count` @@ -64,41 +68,107 @@ module Gitlab # 5 | 1 | 3 # 9 | 2 | 3 # 15 | 3 | 3 + # + # If a partition column is used we will do the same operation but for separate partitions, + # when that happens the CTE might look like this: + # + # project_id | star_count | row_id | ct + # ------------+------------+--------+---- + # 1 | 5 | 1 | 2 + # 1 | 9 | 2 | 2 + # 2 | 10 | 1 | 3 + # 2 | 15 | 2 | 3 + # 2 | 20 | 3 | 3 cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( cte_table, - arel_table - .project( - arel_table[column_sym].as(column_sym.to_s), - Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), - Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), - arel_table.project("COUNT(1)").as('ct')). + arel_table.project(*rank_rows(arel_table, column_sym, partition_column)). # Disallow negative values where(arel_table[column_sym].gteq(zero_interval))) # From the CTE, select either the middle row or the middle two rows (this is accomplished # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the # selected rows, and this is the median value. - cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")) - .where( - Arel::Nodes::Between.new( - cte_table[:row_id], - Arel::Nodes::And.new( - [(cte_table[:ct] / Arel.sql('2.0')), - (cte_table[:ct] / Arel.sql('2.0') + 1)] + result = + cte_table + .project(*median_projections(cte_table, column_sym, partition_column)) + .where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) ) ) - ) - .with(query_so_far, cte) - .to_sql + .with(query_so_far, cte) + + result.group(cte_table[partition_column]).order(cte_table[partition_column]) if partition_column + + result.to_sql end private + def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) + elsif Gitlab::Database.mysql? + raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column + + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + end + + def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) + queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + + Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } + end + def average(args, as) Arel::Nodes::NamedFunction.new("AVG", args, as) end + def rank_rows(arel_table, column_sym, partition_column) + column_row = arel_table[column_sym].as(column_sym.to_s) + + if partition_column + partition_row = arel_table[partition_column] + row_id = + Arel::Nodes::Over.new( + Arel::Nodes::NamedFunction.new('rank', []), + Arel::Nodes::Window.new.partition(arel_table[partition_column]) + .order(arel_table[column_sym]) + ).as('row_id') + + count = arel_table.from(arel_table.alias) + .project('COUNT(*)') + .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) + .as('ct') + + [partition_row, column_row, row_id, count] + else + row_id = + Arel::Nodes::Over.new( + Arel::Nodes::NamedFunction.new('row_number', []), + Arel::Nodes::Window.new.order(arel_table[column_sym]) + ).as('row_id') + + count = arel_table.project("COUNT(1)").as('ct') + + [column_row, row_id, count] + end + end + + def median_projections(table, column_sym, partition_column) + projections = [] + projections << table[partition_column] if partition_column + projections << average([extract_epoch(table[column_sym])], "median") + projections + end + def extract_epoch(arel_attribute) Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dbe6259fce7..21287a8efd0 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -859,6 +859,19 @@ into similar problems in the future (e.g. when new tables are created). BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id]) end end + + def foreign_key_exists?(table, column) + foreign_keys(table).any? do |key| + key.options[:column] == column.to_s + end + end + + # Rails' index_exists? doesn't work when you only give it a table and index + # name. As such we have to use some extra code to check if an index exists for + # a given name. + def index_exists_by_name?(table, index) + indexes(table).map(&:name).include?(index) + end end end end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 88e0db830f6..81df47964be 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -44,7 +44,11 @@ module Gitlab project.commit(head_sha) else straight = start_sha == base_sha - CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + + CompareService.new(project, head_sha).execute(project, + start_sha, + base_sha: base_sha, + straight: straight) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index fcda1fe2233..c358ae428cf 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -13,22 +13,32 @@ module Gitlab end def diff_files - super.tap { |_| store_highlight_cache } + # Make sure to _not_ send any method call to Gitlab::Diff::File + # _before_ all of them were collected (`super`). Premature method calls will + # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy). + # + diff_files = super + + diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) } + store_highlight_cache + + diff_files end def real_size @merge_request_diff.real_size end - private + def clear_cache! + Rails.cache.delete(cache_key) + end - # Extracted method to highlight in the same iteration to the diff_collection. - def decorate_diff!(diff) - diff_file = super - cache_highlight!(diff_file) if cacheable?(diff_file) - diff_file + def cache_key + [@merge_request_diff, 'highlighted-diff-files', diff_options] end + private + def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| Gitlab::Diff::Line.init_from_hash(line) @@ -62,16 +72,12 @@ module Gitlab end def store_highlight_cache - Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty + Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty end def cacheable?(diff_file) @merge_request_diff.present? && diff_file.text? && diff_file.diffable? end - - def cache_key - [@merge_request_diff, 'highlighted-diff-files', diff_options] - end end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index dbb8f317afe..12b5e240962 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -41,6 +41,16 @@ module Gitlab "gitlab:exclusive_lease:#{key}" end + # Removes any existing exclusive_lease from redis + # Don't run this in a live system without making sure no one is using the leases + def self.reset_all!(scope = '*') + Gitlab::Redis::SharedState.with do |redis| + redis.scan_each(match: redis_shared_state_key(scope)).each do |key| + redis.del(key) + end + end + end + def initialize(key, uuid: nil, timeout:) @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b2fca2c16de..eabcf46cf58 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -238,9 +238,9 @@ module Gitlab self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end - @loaded_all_data = false # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data + @loaded_all_data = @loaded_size == size end def binary? @@ -255,10 +255,15 @@ module Gitlab # memory as a Ruby string. def load_all_data!(repository) return if @data == '' # don't mess with submodule blobs - return @data if @loaded_all_data - Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - @data = begin + # Even if we return early, recalculate wether this blob is binary in + # case a blob was initialized as text but the full data isn't + @binary = nil + + return if @loaded_all_data + + @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + begin if is_enabled repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else @@ -269,7 +274,6 @@ module Gitlab @loaded_all_data = true @loaded_size = @data.bytesize - @binary = nil end def name diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index ae7e88f0503..6351cfb83e3 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Branch < Ref + STALE_BRANCH_THRESHOLD = 3.months + def self.find(repo, branch_name) if branch_name.is_a?(Gitlab::Git::Branch) branch_name @@ -12,6 +14,18 @@ module Gitlab def initialize(repository, name, target, target_commit) super(repository, name, target, target_commit) end + + def active? + self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago + end + + def stale? + !active? + end + + def state + active? ? :active : :stale + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index ae27a138b7c..93037ed8d90 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -250,6 +250,45 @@ module Gitlab end end + def extract_signature_lazily(repository, commit_id) + BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + commit_ids = items.map { |i| i[:commit_id] } + + signatures = batch_signature_extraction(repository, commit_ids) + + signatures.each do |commit_sha, signature_data| + loader.call({ repository: repository, commit_id: commit_sha }, signature_data) + end + end + end + end + + def batch_signature_extraction(repository, commit_ids) + repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| + if is_enabled + gitaly_batch_signature_extraction(repository, commit_ids) + else + rugged_batch_signature_extraction(repository, commit_ids) + end + end + end + + def gitaly_batch_signature_extraction(repository, commit_ids) + repository.gitaly_commit_client.get_commit_signatures(commit_ids) + end + + def rugged_batch_signature_extraction(repository, commit_ids) + commit_ids.each_with_object({}) do |commit_id, signatures| + signature_data = rugged_extract_signature(repository, commit_id) + next unless signature_data + + signatures[commit_id] = signature_data + end + end + def rugged_extract_signature(repository, commit_id) begin Rugged::Commit.extract_signature(repository.rugged, commit_id) @@ -308,7 +347,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff - Gitlab::GitalyClient.migrate(:commit_patch) do |is_enabled| + Gitlab::GitalyClient.migrate(:commit_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled @repository.gitaly_commit_client.patch(id) else diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index e5a747cb987..a142ed6b2ef 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -63,11 +63,12 @@ module Gitlab end end - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) + def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) tags_option = tags ? '--tags' : '--no-tags' logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(git fetch #{name} --prune --quiet) + cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) + cmd << '--prune' if prune cmd << '--force' if force cmd << tags_option @@ -84,7 +85,7 @@ module Gitlab def push_branches(remote_name, timeout, force, branch_names) logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %w(git push) + cmd = %W(#{Gitlab.config.git.bin_path} push) cmd << '--force' if force cmd += %W(-- #{remote_name}).concat(branch_names) @@ -101,7 +102,7 @@ module Gitlab branches = branch_names.map { |branch_name| ":#{branch_name}" } logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(git push -- #{remote_name}).concat(branches) + cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) success = run(cmd, repository_absolute_path) @@ -142,7 +143,7 @@ module Gitlab end def remove_origin_in_repo - cmd = %w(git remote rm origin) + cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) run(cmd, repository_absolute_path) end @@ -222,7 +223,7 @@ module Gitlab masked_source = mask_password_in_url(source) logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) success = run_with_timeout(cmd, timeout, nil) @@ -265,7 +266,7 @@ module Gitlab FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 48434047fce..b9e5cf258f4 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,6 +7,28 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) + @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + else + git_new_pointers(object_limit, not_in) + end + end + end + + def all_pointers + @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) + else + git_all_pointers + end + end + end + + private + + def git_new_pointers(object_limit, not_in) @new_pointers ||= begin rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| object_ids = object_ids.take(object_limit) if object_limit @@ -16,14 +38,12 @@ module Gitlab end end - def all_pointers + def git_all_pointers rev_list.all_objects(require_path: true) do |object_ids| Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) end end - private - def rev_list Gitlab::Git::RevList.new(@repository, newrev: @newrev) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e3cbf017e55..fbc93542619 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -228,7 +228,7 @@ module Gitlab end def has_local_branches? - gitaly_migrate(:has_local_branches) do |is_enabled| + gitaly_migrate(:has_local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_repository_client.has_local_branches? else @@ -467,7 +467,8 @@ module Gitlab follow: false, skip_merges: false, after: nil, - before: nil + before: nil, + all: false } options = default_options.merge(options) @@ -489,13 +490,16 @@ module Gitlab # Used in gitaly-ruby def raw_log(options) - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end + sha = + unless options[:all] + actual_ref = options[:ref] || root_ref + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + end log_by_shell(sha, options) end @@ -711,7 +715,7 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_migrate(:operation_user_create_branch) do |is_enabled| + gitaly_migrate(:operation_user_create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_add_branch(branch_name, user, target) else @@ -721,7 +725,7 @@ module Gitlab end def add_tag(tag_name, user:, target:, message: nil) - gitaly_migrate(:operation_user_add_tag) do |is_enabled| + gitaly_migrate(:operation_user_add_tag, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_add_tag(tag_name, user: user, target: target, message: message) else @@ -731,7 +735,7 @@ module Gitlab end def rm_branch(branch_name, user:) - gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_operations_client.user_delete_branch(branch_name, user) else @@ -806,7 +810,7 @@ module Gitlab end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:revert) do |is_enabled| + gitaly_migrate(:revert, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| args = { user: user, commit: commit, @@ -872,7 +876,7 @@ module Gitlab # Delete the specified branch from the repository def delete_branch(branch_name) - gitaly_migrate(:delete_branch) do |is_enabled| + gitaly_migrate(:delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_client.delete_branch(branch_name) else @@ -899,7 +903,7 @@ module Gitlab # create_branch("feature") # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") - gitaly_migrate(:create_branch) do |is_enabled| + gitaly_migrate(:create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_client.create_branch(ref, start_point) else @@ -1006,7 +1010,7 @@ module Gitlab end def languages(ref = nil) - Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| + gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_commit_client.languages(ref) else @@ -1032,6 +1036,21 @@ module Gitlab end end + def license_short_name + gitaly_migrate(:license_short_name) do |is_enabled| + if is_enabled + gitaly_repository_client.license_short_name + else + begin + # The licensee gem creates a Rugged object from the path: + # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb + Licensee.license(path).try(:key) + rescue Rugged::Error + end + end + end + end + def with_repo_branch_commit(start_repository, start_branch_name) Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) @@ -1170,15 +1189,9 @@ module Gitlab end def fsck - gitaly_migrate(:git_fsck) do |is_enabled| - msg, status = if is_enabled - gitaly_fsck - else - shell_fsck - end + msg, status = gitaly_repository_client.fsck - raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? - end + raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? end def create_from_bundle(bundle_path) @@ -1414,22 +1427,12 @@ module Gitlab output end - def can_be_merged?(source_sha, target_branch) - gitaly_migrate(:can_be_merged) do |is_enabled| - if is_enabled - gitaly_can_be_merged?(source_sha, find_branch(target_branch).target) - else - rugged_can_be_merged?(source_sha, target_branch) - end - end - end - - def last_commit_id_for_path(sha, path) + def last_commit_for_path(sha, path) gitaly_migrate(:last_commit_for_path) do |is_enabled| if is_enabled - last_commit_for_path_by_gitaly(sha, path).id + last_commit_for_path_by_gitaly(sha, path) else - last_commit_id_for_path_by_shelling_out(sha, path) + last_commit_for_path_by_rugged(sha, path) end end end @@ -1587,14 +1590,6 @@ module Gitlab File.write(File.join(worktree_info_path, 'sparse-checkout'), files) end - def gitaly_fsck - gitaly_repository_client.fsck - end - - def shell_fsck - run_git(%W[--git-dir=#{path} fsck], nice: true) - end - def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit @@ -1701,7 +1696,12 @@ module Gitlab cmd << '--no-merges' if options[:skip_merges] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha + + if options[:all] + cmd += %w[--all --reverse] + else + cmd << sha + end # :path can be a string or an array of strings if options[:path].present? @@ -1872,7 +1872,7 @@ module Gitlab end def last_commit_for_path_by_rugged(sha, path) - sha = last_commit_id_for_path(sha, path) + sha = last_commit_id_for_path_by_shelling_out(sha, path) commit(sha) end @@ -1918,7 +1918,16 @@ module Gitlab cmd << "--before=#{options[:before].iso8601}" if options[:before] cmd << "--max-count=#{options[:max_count]}" if options[:max_count] cmd << "--left-right" if options[:left_right] - cmd += %W[--count #{options[:ref]}] + cmd << '--count' + + cmd << if options[:all] + '--all' + elsif options[:ref] + options[:ref] + else + raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" + end + cmd += %W[-- #{options[:path]}] if options[:path].present? cmd end @@ -2206,7 +2215,7 @@ module Gitlab with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do # Apply diff of the `diff_range` to the worktree diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| stdin.binmode stdin.write(diff) end @@ -2371,14 +2380,6 @@ module Gitlab .map { |c| commit(c) } end - def gitaly_can_be_merged?(their_commit, our_commit) - !gitaly_conflicts_client(our_commit, their_commit).conflicts? - end - - def rugged_can_be_merged?(their_commit, our_commit) - !rugged.merge_commits(our_commit, their_commit).conflicts? - end - def last_commit_for_path_by_gitaly(sha, path) gitaly_commit_client.last_commit_for_path(sha, path) end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index ba6058fd3c9..b6ceb542dd1 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -14,14 +14,14 @@ module Gitlab # Uses rugged for raw objects # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 - def where(repository, sha, path = nil) + def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - repository.gitaly_commit_client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) else - tree_entries_from_rugged(repository, sha, path) + tree_entries_from_rugged(repository, sha, path, recursive) end end end @@ -57,7 +57,22 @@ module Gitlab end end - def tree_entries_from_rugged(repository, sha, path) + def tree_entries_from_rugged(repository, sha, path, recursive) + current_path_entries = get_tree_entries_from_rugged(repository, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) + end + end + + ordered_entries + end + + def get_tree_entries_from_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index ac12271a87e..52b44b9b3c5 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -59,7 +59,7 @@ module Gitlab end def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| if is_enabled gitaly_get_all_pages else @@ -68,9 +68,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039 def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| if is_enabled gitaly_find_page(title: title, version: version, dir: dir) else diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bbdb593d4e2..6400089a22f 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -199,7 +199,7 @@ module Gitlab def check_repository_existence! unless repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + raise NotFoundError, ERROR_MESSAGES[:no_repo] end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c5d3e944f7d..8ca30ffc232 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -119,12 +119,17 @@ module Gitlab # def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) start = Gitlab::Metrics::System.monotonic_time + request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} + @current_call_id ||= SecureRandom.uuid + enforce_gitaly_request_limits(:call) kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + rescue GRPC::Unavailable => ex + handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -133,7 +138,32 @@ module Gitlab gitaly_controller_action_duration_seconds.observe( current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) + + add_call_details(id: @current_call_id, feature: service, duration: duration, request: request_hash) + + @current_call_id = nil + end + + def self.handle_grpc_unavailable!(ex) + status = ex.to_status + raise ex unless status.details == 'Endpoint read failed' + + # There is a bug in grpc 1.8.x that causes a client process to get stuck + # always raising '14:Endpoint read failed'. The only thing that we can + # do to recover is to restart the process. + # + # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + + if Sidekiq.server? + raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) + else + # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. + Process.kill('QUIT', Process.pid) + end + + raise ex end + private_class_method :handle_grpc_unavailable! def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} @@ -229,12 +259,16 @@ module Gitlab feature_stack.unshift(feature) begin start = Gitlab::Metrics::System.monotonic_time + @current_call_id = SecureRandom.uuid + call_details = { id: @current_call_id } yield is_enabled ensure total_time = Gitlab::Metrics::System.monotonic_time - start gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? + + add_call_details(call_details.merge(feature: feature, duration: total_time)) end end end @@ -321,6 +355,22 @@ module Gitlab end end + def self.add_call_details(details) + id = details.delete(:id) + + return unless id && RequestStore.active? && RequestStore.store[:peek_enabled] + + RequestStore.store['gitaly_call_details'] ||= {} + RequestStore.store['gitaly_call_details'][id] ||= {} + RequestStore.store['gitaly_call_details'][id].merge!(details) + end + + def self.list_call_details + return {} unless RequestStore.active? && RequestStore.store[:peek_enabled] + + RequestStore.store['gitaly_call_details'] || {} + end + def self.expected_server_version path = Rails.root.join(SERVER_VERSION_FILE) path.read.chomp diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index dfa0fa43b0f..28554208984 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -45,16 +45,7 @@ module Gitlab response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) - response.flat_map do |message| - message.lfs_pointers.map do |lfs_pointer| - Gitlab::Git::Blob.new( - id: lfs_pointer.oid, - size: lfs_pointer.size, - data: lfs_pointer.data, - binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) - ) - end - end + map_lfs_pointers(response) end def get_blobs(revision_paths, limit = -1) @@ -80,6 +71,50 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end + + def get_new_lfs_pointers(revision, limit, not_in) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + limit: limit || 0 + ) + + if not_in.nil? || not_in == :all + request.not_in_all = true + else + request.not_in_refs += not_in + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + + map_lfs_pointers(response) + end + + def get_all_lfs_pointers(revision) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + + map_lfs_pointers(response) + end + + private + + def map_lfs_pointers(response) + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 269a048cf5d..456a8a1a2d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -105,11 +105,12 @@ module Gitlab entry unless entry.oid.blank? end - def tree_entries(repository, revision, path) + def tree_entries(repository, revision, path, recursive) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), - path: path.present? ? encode_binary(path) : '.' + path: path.present? ? encode_binary(path) : '.', + recursive: recursive ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -133,7 +134,8 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: encode_binary(ref) + revision: encode_binary(ref), + all: !!options[:all] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -268,6 +270,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], + all: !!options[:all], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] @@ -318,6 +321,23 @@ module Gitlab [signature, signed_text] end + def get_commit_signatures(commit_ids) + request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request) + + signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] } + current_commit_id = nil + + response.each do |message| + current_commit_id = message.commit_id if message.commit_id.present? + + signatures[current_commit_id].first << message.signature + signatures[current_commit_id].last << message.signed_text + end + + signatures + end + private def call_commit_diff(request_params, options = {}) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 603457d0664..e1bc2f9ab61 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -41,14 +41,14 @@ module Gitlab end def apply_gitattributes(revision) - request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) + request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, - no_tags: no_tags, timeout: timeout + no_tags: no_tags, timeout: timeout, no_prune: !prune ) if ssh_auth&.ssh_import? @@ -249,6 +249,14 @@ module Gitlab raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? end + + def license_short_name + request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) + + response = GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.fast_timeout) + + response.license_short_name.presence + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 4f160e4a447..a61beafae0d 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -197,10 +197,7 @@ module Gitlab end def github_omniauth_provider - @github_omniauth_provider ||= - Gitlab.config.omniauth.providers - .find { |provider| provider.name == 'github' } - .to_h + @github_omniauth_provider ||= Gitlab::Auth::OAuth::Provider.config_for('github').to_h end def rate_limit_counter diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 075b3982608..5482504e72e 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -72,7 +72,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find {|provider| provider.name == "gitlab"} + Gitlab::Auth::OAuth::Provider.config_for('gitlab') end def gitlab_options diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ba04387022d..a7e055ac444 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -19,6 +19,8 @@ module Gitlab gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path + gon.test_env = Rails.env.test? + gon.suggested_label_colors = LabelsHelper.suggested_colors if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 90dd569aaf8..6d2278d0876 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,15 +1,29 @@ module Gitlab module Gpg class Commit + include Gitlab::Utils::StrongMemoize + def initialize(commit) @commit = commit repo = commit.project.repository.raw_repository - @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha) + @signature_data = Gitlab::Git::Commit.extract_signature_lazily(repo, commit.sha || commit.id) + end + + def signature_text + strong_memoize(:signature_text) do + @signature_data&.itself && @signature_data[0] + end + end + + def signed_text + strong_memoize(:signed_text) do + @signature_data&.itself && @signature_data[1] + end end def has_signature? - !!(@signature_text && @signed_text) + !!(signature_text && signed_text) end def signature @@ -53,7 +67,7 @@ module Gitlab end def verified_signature - @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| break verified_signature end end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index 1a2eab0b005..d62d9136886 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,3 +1,3 @@ -module Gitlab::HealthChecks +module Gitlab::HealthChecks # rubocop:disable Naming/FileName Metric = Struct.new(:name, :value, :labels) end diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index 8086760023e..e323e2c9723 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,3 +1,3 @@ -module Gitlab::HealthChecks +module Gitlab::HealthChecks # rubocop:disable Naming/FileName Result = Struct.new(:success, :message, :labels) end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index bdc0f04b56b..3772ef11c7f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -18,7 +18,10 @@ module Gitlab 'uk' => 'ะฃะบัะฐัะฝััะบะฐ', 'ja' => 'ๆฅๆฌ่ช', 'ko' => 'ํ๊ตญ์ด', - 'nl_NL' => 'Nederlands' + 'nl_NL' => 'Nederlands', + 'tr_TR' => 'Tรผrkรงe', + 'id_ID' => 'Bahasa Indonesia', + 'fil_PH' => 'Filipino' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9f404003125..4bdd01f5e94 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -65,6 +65,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :project_badges # Only include the following attributes for the models specified. included_attributes: @@ -125,6 +126,8 @@ excluded_attributes: - :when push_event_payload: - :event_id + project_badges: + - :group_id methods: labels: @@ -147,3 +150,5 @@ methods: - :action push_event_payload: - :action + project_badges: + - :type diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a00795f553e..c38df9102eb 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab @archive_file = project.import_source @current_user = project.creator @project = project - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + @shared = project.import_export_shared end def execute diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 759833a5ee5..791a54e1b69 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -16,7 +16,8 @@ module Gitlab priorities: :label_priorities, auto_devops: :project_auto_devops, label: :project_label, - custom_attributes: 'ProjectCustomAttribute' }.freeze + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze @@ -69,6 +70,7 @@ module Gitlab update_user_references update_project_references + remove_duplicate_assignees reset_tokens! remove_encrypted_attributes! @@ -82,6 +84,14 @@ module Gitlab end end + def remove_duplicate_assignees + return unless @relation_hash['issue_assignees'] + + # When an assignee did not exist in the members mapper, the importer is + # assigned. We only need to assign each user once. + @relation_hash['issue_assignees'].uniq!(&:user_id) + end + def setup_note set_note_author # attachment is deprecated and note uploads are handled by Markdown uploader diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index b34cafc6876..3d3d998a6a3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,13 +1,17 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts + attr_reader :errors, :project - def initialize(opts) - @opts = opts + def initialize(project) + @project = project @errors = [] end + def active_export_count + Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) } + end + def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -31,11 +35,11 @@ module Gitlab private def relative_path - File.join(opts[:relative_path], SecureRandom.hex) + File.join(relative_archive_path, SecureRandom.hex) end def relative_archive_path - File.join(opts[:relative_path], '..') + @project.disk_path end def error_out(message, caller) diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index f654508c391..f7a8eae0be4 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -15,16 +15,22 @@ module Gitlab # push to that array when done. Once the waiter has popped `count` items, it # knows all the jobs are done. class JobWaiter + KEY_PREFIX = "gitlab:job_waiter".freeze + def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end + def self.key?(key) + key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ + end + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] diff --git a/lib/gitlab/kubernetes/config_map.rb b/lib/gitlab/kubernetes/config_map.rb new file mode 100644 index 00000000000..95e1054919d --- /dev/null +++ b/lib/gitlab/kubernetes/config_map.rb @@ -0,0 +1,37 @@ +module Gitlab + module Kubernetes + class ConfigMap + def initialize(name, values) + @name = name + @values = values + end + + def generate + resource = ::Kubeclient::Resource.new + resource.metadata = metadata + resource.data = { values: values } + resource + end + + private + + attr_reader :name, :values + + def metadata + { + name: config_map_name, + namespace: namespace, + labels: { name: config_map_name } + } + end + + def config_map_name + "values-content-configuration-#{name}" + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 737081ddc5b..2edd34109ba 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -9,7 +9,8 @@ module Gitlab def install(command) @namespace.ensure_exists! - @kubeclient.create_pod(pod_resource(command)) + create_config_map(command) if command.config_map? + @kubeclient.create_pod(command.pod_resource) end ## @@ -33,8 +34,10 @@ module Gitlab private - def pod_resource(command) - Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate + def create_config_map(command) + command.config_map_resource.tap do |config_map_resource| + @kubeclient.create_config_map(config_map_resource) + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb new file mode 100644 index 00000000000..6e4df05aa7e --- /dev/null +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -0,0 +1,40 @@ +module Gitlab + module Kubernetes + module Helm + class BaseCommand + attr_reader :name + + def initialize(name) + @name = name + end + + def pod_resource + Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate + end + + def generate_script + <<~HEREDOC + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + HEREDOC + end + + def config_map? + false + end + + def pod_name + "install-#{name}" + end + + private + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb new file mode 100644 index 00000000000..a02e64561f6 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -0,0 +1,19 @@ +module Gitlab + module Kubernetes + module Helm + class InitCommand < BaseCommand + def generate_script + super + [ + init_helm_command + ].join("\n") + end + + private + + def init_helm_command + "helm init >/dev/null" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index bf6981035f4..30af3e97b4a 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -1,54 +1,45 @@ module Gitlab module Kubernetes module Helm - class InstallCommand - attr_reader :name, :install_helm, :chart, :chart_values_file + class InstallCommand < BaseCommand + attr_reader :name, :chart, :repository, :values - def initialize(name, install_helm: false, chart: false, chart_values_file: false) + def initialize(name, chart:, values:, repository: nil) @name = name - @install_helm = install_helm @chart = chart - @chart_values_file = chart_values_file + @values = values + @repository = repository end - def pod_name - "install-#{name}" + def generate_script + super + [ + init_command, + repository_command, + script_command + ].compact.join("\n") end - def generate_script(namespace_name) - [ - install_dps_command, - init_command, - complete_command(namespace_name) - ].join("\n") + def config_map? + true + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, values).generate end private def init_command - if install_helm - 'helm init >/dev/null' - else - 'helm init --client-only >/dev/null' - end + 'helm init --client-only >/dev/null' end - def complete_command(namespace_name) - return unless chart - - if chart_values_file - "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null" - else - "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" - end + def repository_command + "helm repo add #{name} #{repository}" if repository end - def install_dps_command + def script_command <<~HEREDOC - set -eo pipefail - apk add -U ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ + helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null HEREDOC end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index ca5e06009fa..1e12299eefd 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -2,18 +2,17 @@ module Gitlab module Kubernetes module Helm class Pod - def initialize(command, namespace_name, kubeclient) + def initialize(command, namespace_name) @command = command @namespace_name = namespace_name - @kubeclient = kubeclient end def generate spec = { containers: [container_specification], restartPolicy: 'Never' } - if command.chart_values_file - create_config_map + if command.config_map? spec[:volumes] = volumes_specification + spec[:containers][0][:volumeMounts] = volume_mounts_specification end ::Kubeclient::Resource.new(metadata: metadata, spec: spec) @@ -21,18 +20,16 @@ module Gitlab private - attr_reader :command, :namespace_name, :kubeclient + attr_reader :command, :namespace_name, :kubeclient, :config_map def container_specification - container = { + { name: 'helm', image: 'alpine:3.6', env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) } - container[:volumeMounts] = volume_mounts_specification if command.chart_values_file - container end def labels @@ -50,13 +47,12 @@ module Gitlab } end - def volume_mounts_specification - [ - { - name: 'configuration-volume', - mountPath: "/data/helm/#{command.name}/config" - } - ] + def generate_pod_env(command) + { + HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, + TILLER_NAMESPACE: namespace_name, + COMMAND_SCRIPT: command.generate_script + }.map { |key, value| { name: key, value: value } } end def volumes_specification @@ -71,23 +67,13 @@ module Gitlab ] end - def generate_pod_env(command) - { - HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, - TILLER_NAMESPACE: namespace_name, - COMMAND_SCRIPT: command.generate_script(namespace_name) - }.map { |key, value| { name: key, value: value } } - end - - def create_config_map - resource = ::Kubeclient::Resource.new - resource.metadata = { - name: "values-content-configuration-#{command.name}", - namespace: namespace_name, - labels: { name: "values-content-configuration-#{command.name}" } - } - resource.data = { values: File.read(command.chart_values_file) } - kubeclient.create_config_map(resource) + def volume_mounts_specification + [ + { + name: 'configuration-volume', + mountPath: "/data/helm/#{command.name}/config" + } + ] end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb deleted file mode 100644 index e60ceba27c8..00000000000 --- a/lib/gitlab/ldap/access.rb +++ /dev/null @@ -1,87 +0,0 @@ -# LDAP authorization model -# -# * Check if we are allowed access (not blocked) -# -module Gitlab - module LDAP - class Access - attr_reader :provider, :user - - def self.open(user, &block) - Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| - block.call(self.new(user, adapter)) - end - end - - def self.allowed?(user) - self.open(user) do |access| - if access.allowed? - Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute - - true - else - false - end - end - end - - def initialize(user, adapter = nil) - @adapter = adapter - @user = user - @provider = user.ldap_identity.provider - end - - def allowed? - if ldap_user - unless ldap_config.active_directory - unblock_user(user, 'is available again') if user.ldap_blocked? - return true - end - - # Block user in GitLab if he/she was blocked in AD - if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - block_user(user, 'is disabled in Active Directory') - false - else - unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? - true - end - else - # Block the user if they no longer exist in LDAP/AD - block_user(user, 'does not exist anymore') - false - end - end - - def adapter - @adapter ||= Gitlab::LDAP::Adapter.new(provider) - end - - def ldap_config - Gitlab::LDAP::Config.new(provider) - end - - def ldap_user - @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) - end - - def block_user(user, reason) - user.ldap_block - - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" - ) - end - - def unblock_user(user, reason) - user.activate - - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" - ) - end - end - end -end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb deleted file mode 100644 index 76863e77dc3..00000000000 --- a/lib/gitlab/ldap/adapter.rb +++ /dev/null @@ -1,108 +0,0 @@ -module Gitlab - module LDAP - class Adapter - attr_reader :provider, :ldap - - def self.open(provider, &block) - Net::LDAP.open(config(provider).adapter_options) do |ldap| - block.call(self.new(provider, ldap)) - end - end - - def self.config(provider) - Gitlab::LDAP::Config.new(provider) - end - - def initialize(provider, ldap = nil) - @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) - end - - def config - Gitlab::LDAP::Config.new(provider) - end - - def users(fields, value, limit = nil) - options = user_options(Array(fields), value, limit) - - entries = ldap_search(options).select do |entry| - entry.respond_to? config.uid - end - - entries.map do |entry| - Gitlab::LDAP::Person.new(entry, provider) - end - end - - def user(*args) - users(*args).first - end - - def dn_matches_filter?(dn, filter) - ldap_search(base: dn, - filter: filter, - scope: Net::LDAP::SearchScope_BaseObject, - attributes: %w{dn}).any? - end - - def ldap_search(*args) - # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do - results = ldap.search(*args) - - if results.nil? - response = ldap.get_operation_result - - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end - - [] - else - results - end - end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] - end - - private - - def user_options(fields, value, limit) - options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config), - base: config.base - } - - options[:size] = limit if limit - - if fields.include?('dn') - raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 - - options[:base] = value - options[:scope] = Net::LDAP::SearchScope_BaseObject - else - filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) - end - - options.merge(filter: user_filter(filter)) - end - - def user_filter(filter = nil) - user_filter = config.constructed_user_filter if config.user_filter.present? - - if user_filter && filter - Net::LDAP::Filter.join(filter, user_filter) - elsif user_filter - user_filter - else - filter - end - end - end - end -end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb deleted file mode 100644 index 96171dc26c4..00000000000 --- a/lib/gitlab/ldap/auth_hash.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Class to parse and transform the info provided by omniauth -# -module Gitlab - module LDAP - class AuthHash < Gitlab::OAuth::AuthHash - def uid - @uid ||= Gitlab::LDAP::Person.normalize_dn(super) - end - - def username - super.tap do |username| - username.downcase! if ldap_config.lowercase_usernames - end - end - - private - - def get_info(key) - attributes = ldap_config.attributes[key.to_s] - return super unless attributes - - attributes = Array(attributes) - - value = nil - attributes.each do |attribute| - value = get_raw(attribute) - value = value.first if value - break if value.present? - end - - return super unless value - - Gitlab::Utils.force_utf8(value) - value - end - - def get_raw(key) - auth_hash.extra[:raw_info][key] if auth_hash.extra - end - - def ldap_config - @ldap_config ||= Gitlab::LDAP::Config.new(self.provider) - end - end - end -end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb deleted file mode 100644 index 7274d1c3b43..00000000000 --- a/lib/gitlab/ldap/authentication.rb +++ /dev/null @@ -1,70 +0,0 @@ -# These calls help to authenticate to LDAP by providing username and password -# -# Since multiple LDAP servers are supported, it will loop through all of them -# until a valid bind is found -# - -module Gitlab - module LDAP - class Authentication - def self.login(login, password) - return unless Gitlab::LDAP::Config.enabled? - return unless login.present? && password.present? - - auth = nil - # loop through providers until valid bind - providers.find do |provider| - auth = new(provider) - auth.login(login, password) # true will exit the loop - end - - # If (login, password) was invalid for all providers, the value of auth is now the last - # Gitlab::LDAP::Authentication instance we tried. - auth.user - end - - def self.providers - Gitlab::LDAP::Config.providers - end - - attr_accessor :provider, :ldap_user - - def initialize(provider) - @provider = provider - end - - def login(login, password) - @ldap_user = adapter.bind_as( - filter: user_filter(login), - size: 1, - password: password - ) - end - - def adapter - OmniAuth::LDAP::Adaptor.new(config.omniauth_options) - end - - def config - Gitlab::LDAP::Config.new(provider) - end - - def user_filter(login) - filter = Net::LDAP::Filter.equals(config.uid, login) - - # Apply LDAP user filter if present - if config.user_filter.present? - filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) - end - - filter - end - - def user - return nil unless ldap_user - - Gitlab::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) - end - end - end -end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb deleted file mode 100644 index a6bea98d631..00000000000 --- a/lib/gitlab/ldap/config.rb +++ /dev/null @@ -1,235 +0,0 @@ -# Load a specific server configuration -module Gitlab - module LDAP - class Config - NET_LDAP_ENCRYPTION_METHOD = { - simple_tls: :simple_tls, - start_tls: :start_tls, - plain: nil - }.freeze - - attr_accessor :provider, :options - - def self.enabled? - Gitlab.config.ldap.enabled - end - - def self.servers - Gitlab.config.ldap['servers']&.values || [] - end - - def self.available_servers - return [] unless enabled? - - Array.wrap(servers.first) - end - - def self.providers - servers.map { |server| server['provider_name'] } - end - - def self.valid_provider?(provider) - providers.include?(provider) - end - - def self.invalid_provider(provider) - raise "Unknown provider (#{provider}). Available providers: #{providers}" - end - - def initialize(provider) - if self.class.valid_provider?(provider) - @provider = provider - else - self.class.invalid_provider(provider) - end - - @options = config_for(@provider) # Use @provider, not provider - end - - def enabled? - base_config.enabled - end - - def adapter_options - opts = base_options.merge( - encryption: encryption_options - ) - - opts.merge!(auth_options) if has_auth? - - opts - end - - def omniauth_options - opts = base_options.merge( - base: base, - encryption: options['encryption'], - filter: omniauth_user_filter, - name_proc: name_proc, - disable_verify_certificates: !options['verify_certificates'] - ) - - if has_auth? - opts.merge!( - bind_dn: options['bind_dn'], - password: options['password'] - ) - end - - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - - opts - end - - def base - options['base'] - end - - def uid - options['uid'] - end - - def sync_ssh_keys? - sync_ssh_keys.present? - end - - # The LDAP attribute in which the ssh keys are stored - def sync_ssh_keys - options['sync_ssh_keys'] - end - - def user_filter - options['user_filter'] - end - - def constructed_user_filter - @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) - end - - def group_base - options['group_base'] - end - - def admin_group - options['admin_group'] - end - - def active_directory - options['active_directory'] - end - - def block_auto_created_users - options['block_auto_created_users'] - end - - def attributes - default_attributes.merge(options['attributes']) - end - - def timeout - options['timeout'].to_i - end - - def has_auth? - options['password'] || options['bind_dn'] - end - - def allow_username_or_email_login - options['allow_username_or_email_login'] - end - - def lowercase_usernames - options['lowercase_usernames'] - end - - def name_proc - if allow_username_or_email_login - proc { |name| name.gsub(/@.*\z/, '') } - else - proc { |name| name } - end - end - - def default_attributes - { - 'username' => %w(uid sAMAccountName userid), - 'email' => %w(mail email userPrincipalName), - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' - } - end - - protected - - def base_options - { - host: options['host'], - port: options['port'] - } - end - - def base_config - Gitlab.config.ldap - end - - def config_for(provider) - base_config.servers.values.find { |server| server['provider_name'] == provider } - end - - def encryption_options - method = translate_method(options['encryption']) - return nil unless method - - { - method: method, - tls_options: tls_options(method) - } - end - - def translate_method(method_from_config) - NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] - end - - def tls_options(method) - return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method - - opts = if options['verify_certificates'] - OpenSSL::SSL::SSLContext::DEFAULT_PARAMS - else - # It is important to explicitly set verify_mode for two reasons: - # 1. The behavior of OpenSSL is undefined when verify_mode is not set. - # 2. The net-ldap gem implementation verifies the certificate hostname - # unless verify_mode is set to VERIFY_NONE. - { verify_mode: OpenSSL::SSL::VERIFY_NONE } - end - - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - - opts - end - - def auth_options - { - auth: { - method: :simple, - username: options['bind_dn'], - password: options['password'] - } - } - end - - def omniauth_user_filter - uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') - - if user_filter.present? - Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s - else - uid_filter.to_s - end - end - end - end -end diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb deleted file mode 100644 index d6142dc6549..00000000000 --- a/lib/gitlab/ldap/dn.rb +++ /dev/null @@ -1,301 +0,0 @@ -# -*- ruby encoding: utf-8 -*- - -# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` -# -# For our purposes, this class is used to normalize DNs in order to allow proper -# comparison. -# -# E.g. DNs should be compared case-insensitively (in basically all LDAP -# implementations or setups), therefore we downcase every DN. - -## -# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN -# ("Distinguished Name") is a unique identifier for an entry within an LDAP -# directory. It is made up of a number of other attributes strung together, -# to identify the entry in the tree. -# -# Each attribute that makes up a DN needs to have its value escaped so that -# the DN is valid. This class helps take care of that. -# -# A fully escaped DN needs to be unescaped when analysing its contents. This -# class also helps take care of that. -module Gitlab - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) - - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end - - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) - end - end - - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" - - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char - end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" - end - end - - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state - - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end - - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) - - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! - end - end - - str - end - - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end - - ## - # Return the DN as an escaped string. - def to_s - @dn - end - - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end - - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze - - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") - - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") - - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end - - private - - def initialize_array(args) - buffer = StringIO.new - - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) - end - end - - @dn = buffer.string - end - - def initialize_string(arg) - @dn = arg.to_s - end - - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end - - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) - end - end - end -end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb deleted file mode 100644 index c59df556247..00000000000 --- a/lib/gitlab/ldap/person.rb +++ /dev/null @@ -1,120 +0,0 @@ -module Gitlab - module LDAP - class Person - # Active Directory-specific LDAP filter that checks if bit 2 of the - # userAccountControl attribute is set. - # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ - AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") - - InvalidEntryError = Class.new(StandardError) - - attr_accessor :entry, :provider - - def self.find_by_uid(uid, adapter) - uid = Net::LDAP::Filter.escape(uid) - adapter.user(adapter.config.uid, uid) - end - - def self.find_by_dn(dn, adapter) - adapter.user('dn', dn) - end - - def self.find_by_email(email, adapter) - email_fields = adapter.config.attributes['email'] - - adapter.user(email_fields, email) - end - - def self.disabled_via_active_directory?(dn, adapter) - adapter.dn_matches_filter?(dn, AD_USER_DISABLED) - end - - def self.ldap_attributes(config) - [ - 'dn', - config.uid, - *config.attributes['name'], - *config.attributes['email'], - *config.attributes['username'] - ].compact.uniq - end - - def self.normalize_dn(dn) - ::Gitlab::LDAP::DN.new(dn).to_normalized_s - rescue ::Gitlab::LDAP::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") - - dn - end - - # Returns the UID in a normalized form. - # - # 1. Excess spaces are stripped - # 2. The string is downcased (for case-insensitivity) - def self.normalize_uid(uid) - ::Gitlab::LDAP::DN.normalize_value(uid) - rescue ::Gitlab::LDAP::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") - - uid - end - - def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } - @entry = entry - @provider = provider - end - - def name - attribute_value(:name).first - end - - def uid - entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend - end - - def username - username = attribute_value(:username) - - # Depending on the attribute, multiple values may - # be returned. We need only one for username. - # Ex. `uid` returns only one value but `mail` may - # return an array of multiple email addresses. - [username].flatten.first.tap do |username| - username.downcase! if config.lowercase_usernames - end - end - - def email - attribute_value(:email) - end - - def dn - self.class.normalize_dn(entry.dn) - end - - private - - def entry - @entry - end - - def config - @config ||= Gitlab::LDAP::Config.new(provider) - end - - # Using the LDAP attributes configuration, find and return the first - # attribute with a value. For example, by default, when given 'email', - # this method looks for 'mail', 'email' and 'userPrincipalName' and - # returns the first with a value. - def attribute_value(attribute) - attributes = Array(config.attributes[attribute.to_s]) - selected_attr = attributes.find { |attr| entry.respond_to?(attr) } - - return nil unless selected_attr - - entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend - end - end - end -end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb deleted file mode 100644 index 84ee94e38e4..00000000000 --- a/lib/gitlab/ldap/user.rb +++ /dev/null @@ -1,52 +0,0 @@ -# LDAP extension for User model -# -# * Find or create user from omniauth.auth data -# * Links LDAP account with existing user -# * Auth LDAP user with login and password -# -module Gitlab - module LDAP - class User < Gitlab::OAuth::User - class << self - def find_by_uid_and_provider(uid, provider) - identity = ::Identity.with_extern_uid(provider, uid).take - - identity && identity.user - end - end - - def save - super('LDAP') - end - - # instance methods - def find_user - find_by_uid_and_provider || find_by_email || build_new_user - end - - def find_by_uid_and_provider - self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) - end - - def changed? - gl_user.changed? || gl_user.identities.any?(&:changed?) - end - - def block_after_signup? - ldap_config.block_auto_created_users - end - - def allowed? - Gitlab::LDAP::Access.allowed?(gl_user) - end - - def ldap_config - Gitlab::LDAP::Config.new(auth_hash.provider) - end - - def auth_hash=(auth_hash) - @auth_hash = Gitlab::LDAP::AuthHash.new(auth_hash) - end - end - end -end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index 53c910d44bd..d8ed0ebca9d 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -83,7 +83,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + Gitlab::Auth::OAuth::Provider.config_for('github') end def github_options diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index cbabe5454ca..3ce245a8050 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -12,9 +12,8 @@ module Gitlab @type = type end - def execute - ::Projects::CreateService.new( - current_user, + def execute(extra_attrs = {}) + attrs = { name: name, path: name, description: repo.description, @@ -24,7 +23,9 @@ module Gitlab import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki - ).execute + }.merge!(extra_attrs) + + ::Projects::CreateService.new(current_user, attrs).execute end private diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..d9d5f90596f 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,90 +1,19 @@ module Gitlab module Middleware class ReadOnly - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze API_VERSIONS = (3..4) + def self.internal_routes + @internal_routes ||= + API_VERSIONS.map { |version| "api/v#{version}/internal" } + end + def initialize(app) @app = app - @whitelisted = internal_routes end def call(env) - @env = env - @route_hash = nil - - if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') - error_message = 'You cannot do writing operations on a read-only GitLab instance' - - if json_request? - return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] - else - rack_flash.alert = error_message - rack_session['flash'] = rack_flash.to_session_value - - return [301, { 'Location' => last_visited_url }, []] - end - end - - @app.call(env) - end - - private - - def internal_routes - API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } - end - - def disallowed_request? - DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes - end - - def json_request? - request.media_type == APPLICATION_JSON - end - - def rack_flash - @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) - end - - def rack_session - @env['rack.session'] - end - - def request - @env['rack.request'] ||= Rack::Request.new(@env) - end - - def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url - end - - def route_hash - @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} - end - - def whitelisted_routes - grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - - def grack_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') - - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' - end - - def lfs_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') - - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb new file mode 100644 index 00000000000..45b644e6510 --- /dev/null +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -0,0 +1,86 @@ +module Gitlab + module Middleware + class ReadOnly + class Controller + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + + def initialize(app, env) + @app = app + @env = env + end + + def call + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + + if json_request? + return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] + else + rack_flash.alert = ERROR_MESSAGE + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(@env) + end + + private + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && + !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end + end +end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb new file mode 100644 index 00000000000..bfe8e113b5e --- /dev/null +++ b/lib/gitlab/middleware/release_env.rb @@ -0,0 +1,14 @@ +module Gitlab # rubocop:disable Naming/FileName + module Middleware + # Some of middleware would hold env for no good reason even after the + # request had already been processed, and we could not garbage collect + # them due to this. Put this middleware as the first middleware so that + # it would clear the env after the request is done, allowing GC gets a + # chance to release memory for the last request. + ReleaseEnv = Struct.new(:app) do + def call(env) + app.call(env).tap { env.clear } + end + end + end +end diff --git a/lib/gitlab/o_auth.rb b/lib/gitlab/o_auth.rb deleted file mode 100644 index 5ad8d83bd6e..00000000000 --- a/lib/gitlab/o_auth.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module OAuth - SignupDisabledError = Class.new(StandardError) - SigninDisabledForProviderError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb deleted file mode 100644 index 5b5ed449f94..00000000000 --- a/lib/gitlab/o_auth/auth_hash.rb +++ /dev/null @@ -1,90 +0,0 @@ -# Class to parse and transform the info provided by omniauth -# -module Gitlab - module OAuth - class AuthHash - attr_reader :auth_hash - def initialize(auth_hash) - @auth_hash = auth_hash - end - - def uid - @uid ||= Gitlab::Utils.force_utf8(auth_hash.uid.to_s) - end - - def provider - @provider ||= auth_hash.provider.to_s - end - - def name - @name ||= get_info(:name) || "#{get_info(:first_name)} #{get_info(:last_name)}" - end - - def username - @username ||= username_and_email[:username].to_s - end - - def email - @email ||= username_and_email[:email].to_s - end - - def password - @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) - end - - def location - location = get_info(:address) - if location.is_a?(Hash) - [location.locality.presence, location.country.presence].compact.join(', ') - else - location - end - end - - def has_attribute?(attribute) - if attribute == :location - get_info(:address).present? - else - get_info(attribute).present? - end - end - - private - - def info - auth_hash.info - end - - def get_info(key) - value = info[key] - Gitlab::Utils.force_utf8(value) if value - value - end - - def username_and_email - @username_and_email ||= begin - username = get_info(:username).presence || get_info(:nickname).presence - email = get_info(:email).presence - - username ||= generate_username(email) if email - email ||= generate_temporarily_email(username) if username - - { - username: username, - email: email - } - end - end - - # Get the first part of the email address (before @) - # In addtion in removes illegal characters - def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s - end - - def generate_temporarily_email(username) - "temp-email-for-oauth-#{username}@gitlab.localhost" - end - end - end -end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb deleted file mode 100644 index 657db29c85a..00000000000 --- a/lib/gitlab/o_auth/provider.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Gitlab - module OAuth - class Provider - LABELS = { - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google" - }.freeze - - def self.providers - Devise.omniauth_providers - end - - def self.enabled?(name) - providers.include?(name.to_sym) - end - - def self.ldap_provider?(name) - name.to_s.start_with?('ldap') - end - - def self.sync_profile_from_provider?(provider) - return true if ldap_provider?(provider) - - providers = Gitlab.config.omniauth.sync_profile_from_provider - - if providers.is_a?(Array) - providers.include?(provider) - else - providers - end - end - - def self.config_for(name) - name = name.to_s - if ldap_provider?(name) - if Gitlab::LDAP::Config.valid_provider?(name) - Gitlab::LDAP::Config.new(name).options - else - nil - end - else - Gitlab.config.omniauth.providers.find { |provider| provider.name == name } - end - end - - def self.label_for(name) - name = name.to_s - config = config_for(name) - (config && config['label']) || LABELS[name] || name.titleize - end - end - end -end diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb deleted file mode 100644 index 30739f2a2c5..00000000000 --- a/lib/gitlab/o_auth/session.rb +++ /dev/null @@ -1,19 +0,0 @@ -# :nocov: -module Gitlab - module OAuth - module Session - def self.create(provider, ticket) - Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) - end - - def self.destroy(provider, ticket) - Rails.cache.delete("gitlab:#{provider}:#{ticket}") - end - - def self.valid?(provider, ticket) - Rails.cache.read("gitlab:#{provider}:#{ticket}").present? - end - end - end -end -# :nocov: diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb deleted file mode 100644 index 28ebac1776e..00000000000 --- a/lib/gitlab/o_auth/user.rb +++ /dev/null @@ -1,241 +0,0 @@ -# OAuth extension for User model -# -# * Find GitLab user based on omniauth uid and provider -# * Create new user from omniauth data -# -module Gitlab - module OAuth - class User - attr_accessor :auth_hash, :gl_user - - def initialize(auth_hash) - self.auth_hash = auth_hash - update_profile - add_or_update_user_identities - end - - def persisted? - gl_user.try(:persisted?) - end - - def new? - !persisted? - end - - def valid? - gl_user.try(:valid?) - end - - def save(provider = 'OAuth') - raise SigninDisabledForProviderError if oauth_provider_disabled? - raise SignupDisabledError unless gl_user - - block_after_save = needs_blocking? - - Users::UpdateService.new(gl_user, user: gl_user).execute! - - gl_user.block if block_after_save - - log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" - gl_user - rescue ActiveRecord::RecordInvalid => e - log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" - return self, e.record.errors - end - - def gl_user - return @gl_user if defined?(@gl_user) - - @gl_user = find_user - end - - def find_user - user = find_by_uid_and_provider - - user ||= find_or_build_ldap_user if auto_link_ldap_user? - user ||= build_new_user if signup_enabled? - - user.external = true if external_provider? && user&.new_record? - - user - end - - protected - - def add_or_update_user_identities - return unless gl_user - - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - - identity ||= gl_user.identities.build(provider: auth_hash.provider) - identity.extern_uid = auth_hash.uid - - if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person - log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." - gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) - end - end - - def find_or_build_ldap_user - return unless ldap_person - - user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) - if user - log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - return user - end - - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - build_new_user - end - - def find_by_email - return unless auth_hash.has_attribute?(:email) - - ::User.find_by(email: auth_hash.email.downcase) - end - - def auto_link_ldap_user? - Gitlab.config.omniauth.auto_link_ldap_user - end - - def creating_linked_ldap_user? - auto_link_ldap_user? && ldap_person - end - - def ldap_person - return @ldap_person if defined?(@ldap_person) - - # Look for a corresponding person with same uid in any of the configured LDAP providers - Gitlab::LDAP::Config.providers.each do |provider| - adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = find_ldap_person(auth_hash, adapter) - break if @ldap_person - end - @ldap_person - end - - def find_ldap_person(auth_hash, adapter) - Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || - Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || - Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) - end - - def ldap_config - Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person - end - - def needs_blocking? - new? && block_after_signup? - end - - def signup_enabled? - providers = Gitlab.config.omniauth.allow_single_sign_on - if providers.is_a?(Array) - providers.include?(auth_hash.provider) - else - providers - end - end - - def external_provider? - Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) - end - - def block_after_signup? - if creating_linked_ldap_user? - ldap_config.block_auto_created_users - else - Gitlab.config.omniauth.block_auto_created_users - end - end - - def auth_hash=(auth_hash) - @auth_hash = AuthHash.new(auth_hash) - end - - def find_by_uid_and_provider - identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take - identity && identity.user - end - - def build_new_user - user_params = user_attributes.merge(skip_confirmation: true) - Users::BuildService.new(nil, user_params).execute(skip_authorization: true) - end - - def user_attributes - # Give preference to LDAP for sensitive information when creating a linked account - if creating_linked_ldap_user? - username = ldap_person.username.presence - email = ldap_person.email.first.presence - end - - username ||= auth_hash.username - email ||= auth_hash.email - - valid_username = ::Namespace.clean_path(username) - - uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } - - name = auth_hash.name - name = valid_username if name.strip.empty? - - { - name: name, - username: valid_username, - email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, - password_automatically_set: true - } - end - - def sync_profile_from_provider? - Gitlab::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) - end - - def update_profile - clear_user_synced_attributes_metadata - - return unless sync_profile_from_provider? || creating_linked_ldap_user? - - metadata = gl_user.build_user_synced_attributes_metadata - - if sync_profile_from_provider? - UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| - if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) - gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend - metadata.set_attribute_synced(key, true) - else - metadata.set_attribute_synced(key, false) - end - end - - metadata.provider = auth_hash.provider - end - - if creating_linked_ldap_user? && gl_user.email == ldap_person.email.first - metadata.set_attribute_synced(:email, true) - metadata.provider = ldap_person.provider - end - end - - def clear_user_synced_attributes_metadata - gl_user&.user_synced_attributes_metadata&.destroy - end - - def log - Gitlab::AppLogger - end - - def oauth_provider_disabled? - Gitlab::CurrentSettings.current_application_settings - .disabled_oauth_sign_in_sources - .include?(auth_hash.provider) - end - end - end -end diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/plugin.rb new file mode 100644 index 00000000000..0d1cb16b378 --- /dev/null +++ b/lib/gitlab/plugin.rb @@ -0,0 +1,26 @@ +module Gitlab + module Plugin + def self.files + Dir.glob(Rails.root.join('plugins/*')).select do |entry| + File.file?(entry) + end + end + + def self.execute_all_async(data) + args = files.map { |file| [file, data] } + + PluginWorker.bulk_perform_async(args) + end + + def self.execute(file, data) + result = Gitlab::Popen.popen_with_detail([file]) do |stdin| + stdin.write(data.to_json) + end + + exit_status = result.status&.exitstatus + [exit_status.zero?, result.stderr] + rescue => e + [false, e.message] + end + end +end diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/plugin_logger.rb new file mode 100644 index 00000000000..c4f6ec3e21d --- /dev/null +++ b/lib/gitlab/plugin_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class PluginLogger < Gitlab::Logger + def self.file_name_noext + 'plugin' + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index cf0935dbd9a..29277ec6481 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,8 +29,18 @@ module Gitlab @blobs_count ||= blobs.count end - def notes_count - @notes_count ||= notes.count + def limited_notes_count + return @limited_notes_count if defined?(@limited_notes_count) + + types = %w(issue merge_request commit snippet) + @limited_notes_count = 0 + + types.each do |type| + @limited_notes_count += notes_finder(type).limit(count_limit).count + break if @limited_notes_count >= count_limit + end + + @limited_notes_count end def wiki_blobs_count @@ -72,11 +82,12 @@ module Gitlab end def single_commit_result? - commits_count == 1 && total_result_count == 1 - end + return false if commits_count != 1 - def total_result_count - issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + counts = %i(limited_milestones_count limited_notes_count + limited_merge_requests_count limited_issues_count + blobs_count wiki_blobs_count) + counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend end private @@ -106,7 +117,11 @@ module Gitlab end def notes - @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') + @notes ||= notes_finder(nil) + end + + def notes_finder(type) + NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb index 1bba0b78e2f..690c38737c0 100644 --- a/lib/gitlab/project_transfer.rb +++ b/lib/gitlab/project_transfer.rb @@ -1,13 +1,19 @@ module Gitlab + # This class is used to move local, unhashed files owned by projects to their new location class ProjectTransfer - def move_project(project_path, namespace_path_was, namespace_path) - new_namespace_folder = File.join(root_dir, namespace_path) - FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) - from = File.join(root_dir, namespace_path_was, project_path) - to = File.join(root_dir, namespace_path, project_path) + # nil parent_path (or parent_path_was) represents a root namespace + def move_namespace(path, parent_path_was, parent_path) + parent_path_was ||= '' + parent_path ||= '' + new_parent_folder = File.join(root_dir, parent_path) + FileUtils.mkdir_p(new_parent_folder) + from = File.join(root_dir, parent_path_was, path) + to = File.join(root_dir, parent_path, path) move(from, to, "") end + alias_method :move_project, :move_namespace + def rename_project(path_was, path, namespace_path) base_dir = File.join(root_dir, namespace_path) move(path_was, path, base_dir) diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index cb95daf2260..bb1172f82a1 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -1,10 +1,12 @@ module Gitlab module Prometheus module AdditionalMetricsParser + CONFIG_ROOT = 'config/prometheus'.freeze + MUTEX = Mutex.new extend self - def load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) + def load_groups_from_yaml(file_name = 'additional_metrics.yml') + yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end private @@ -22,13 +24,20 @@ module Gitlab MetricGroup.new(entry).tap(&method(:validate!)) end - def additional_metrics_raw - load_yaml_file&.map(&:deep_symbolize_keys).freeze + def yaml_metrics_raw(file_name) + load_yaml_file(file_name)&.map(&:deep_symbolize_keys).freeze end - def load_yaml_file - @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def load_yaml_file(file_name) + return YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) if Rails.env.development? + + MUTEX.synchronize do + @loaded_yaml_cache ||= {} + @loaded_yaml_cache[file_name] ||= YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 972ab75d1d5..e677ec84cd4 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( deployment.project, diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index c60828165bd..29cab6e9c15 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -20,6 +20,10 @@ module Gitlab def query(*args) raise NotImplementedError end + + def self.transform_reactive_result(result) + result + end end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 6e6da593178..c2626581897 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +25,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 1d17d3cfd56..b62910c8de6 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -19,6 +19,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index 5710ad47c1a..d920e9a749f 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -1,7 +1,7 @@ module Gitlab module Prometheus module Queries - class MatchedMetricsQuery < BaseQuery + class MatchedMetricQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze def query diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 0c280dc9a3c..aad76e335af 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -3,9 +3,16 @@ module Gitlab module Queries module QueryAdditionalMetrics def query_metrics(project, query_context) + matched_metrics(project).map(&query_group(query_context)) + .select(&method(:group_with_any_metrics)) + end + + protected + + def query_group(query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics(project).map do |group| + lambda do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -21,8 +28,6 @@ module Gitlab metrics: metrics.select(&method(:metric_with_any_queries)) } end - - groups.select(&method(:group_with_any_metrics)) end private @@ -72,12 +77,17 @@ module Gitlab end def common_query_context(environment, timeframe_start:, timeframe_end:) - { - timeframe_start: timeframe_start, - timeframe_end: timeframe_end, + base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + }) + end + + def base_query_context(timeframe_start, timeframe_end) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end } end end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 659021c9ac9..b66253a10e0 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -57,7 +57,11 @@ module Gitlab rescue OpenSSL::SSL::SSLError raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex - handle_exception_response(ex.response) + if ex.response + handle_exception_response(ex.response) + else + raise PrometheusClient::Error, "Network connection error" + end rescue RestClient::Exception raise PrometheusClient::Error, "Network connection error" end diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb new file mode 100644 index 00000000000..b1bf3ca4143 --- /dev/null +++ b/lib/gitlab/repository_cache.rb @@ -0,0 +1,33 @@ +# Interface to the Redis-backed cache store +module Gitlab + class RepositoryCache + attr_reader :repository, :namespace, :backend + + def initialize(repository, extra_namespace: nil, backend: Rails.cache) + @repository = repository + @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace += ":#{extra_namespace}" if extra_namespace + @backend = backend + end + + def cache_key(type) + "#{type}:#{namespace}" + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def fetch(key, &block) + backend.fetch(cache_key(key), &block) + end + + def exist?(key) + backend.exist?(cache_key(key)) + end + + def read(key) + backend.read(cache_key(key)) + end + end +end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb new file mode 100644 index 00000000000..7f64a8c9e46 --- /dev/null +++ b/lib/gitlab/repository_cache_adapter.rb @@ -0,0 +1,84 @@ +module Gitlab + module RepositoryCacheAdapter + extend ActiveSupport::Concern + + class_methods do + # Wraps around the given method and caches its output in Redis and an instance + # variable. + # + # This only works for methods that do not take any arguments. + def cache_method(name, fallback: nil, memoize_only: false) + original = :"_uncached_#{name}" + + alias_method(original, name) + + define_method(name) do + cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do + __send__(original) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + + # RepositoryCache to be used. Should be overridden by the including class + def cache + raise NotImplementedError + end + + # Caches the supplied block both in a cache and in an instance variable. + # + # The cache key and instance variable are named the same way as the value of + # the `key` argument. + # + # This method will return `nil` if the corresponding instance variable is also + # set to `nil`. This ensures we don't keep yielding the block when it returns + # `nil`. + # + # key - The name of the key to cache the data in. + # fallback - A value to fall back to in the event of a Git error. + def cache_method_output(key, fallback: nil, memoize_only: false, &block) + ivar = cache_instance_variable_name(key) + + if instance_variable_defined?(ivar) + instance_variable_get(ivar) + else + # If the repository doesn't exist and a fallback was specified we return + # that value inmediately. This saves us Rugged/gRPC invocations. + return fallback unless fallback.nil? || cache.repository.exists? + + begin + value = + if memoize_only + yield + else + cache.fetch(key, &block) + end + + instance_variable_set(ivar, value) + rescue Gitlab::Git::Repository::NoRepository + # Even if the above `#exists?` check passes these errors might still + # occur (for example because of a non-existing HEAD). We want to + # gracefully handle this and not cache anything + fallback + end + end + end + + # Expires the caches of a specific set of methods + def expire_method_caches(methods) + methods.each do |key| + cache.expire(key) + + ivar = cache_instance_variable_name(key) + + remove_instance_variable(ivar) if instance_variable_defined?(ivar) + end + end + + private + + def cache_instance_variable_name(key) + :"@#{key.to_s.tr('?!', '')}" + end + end +end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb deleted file mode 100644 index 33d19373098..00000000000 --- a/lib/gitlab/saml/auth_hash.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Gitlab - module Saml - class AuthHash < Gitlab::OAuth::AuthHash - def groups - Array.wrap(get_raw(Gitlab::Saml::Config.groups)) - end - - private - - def get_raw(key) - # Needs to call `all` because of https://git.io/vVo4u - # otherwise just the first value is returned - auth_hash.extra[:raw_info].all[key] - end - end - end -end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb deleted file mode 100644 index 574c3a4b28c..00000000000 --- a/lib/gitlab/saml/config.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Gitlab - module Saml - class Config - class << self - def options - Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } - end - - def groups - options[:groups_attribute] - end - - def external_groups - options[:external_groups] - end - end - end - end -end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb deleted file mode 100644 index d8faf7aad8c..00000000000 --- a/lib/gitlab/saml/user.rb +++ /dev/null @@ -1,50 +0,0 @@ -# SAML extension for User model -# -# * Find GitLab user based on SAML uid and provider -# * Create new user from SAML data -# -module Gitlab - module Saml - class User < Gitlab::OAuth::User - def save - super('SAML') - end - - def find_user - user = find_by_uid_and_provider - - user ||= find_by_email if auto_link_saml_user? - user ||= find_or_build_ldap_user if auto_link_ldap_user? - user ||= build_new_user if signup_enabled? - - if external_users_enabled? && user - # Check if there is overlap between the user's groups and the external groups - # setting then set user as external or internal. - user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - end - - user - end - - def changed? - return true unless gl_user - - gl_user.changed? || gl_user.identities.any?(&:changed?) - end - - protected - - def auto_link_saml_user? - Gitlab.config.omniauth.auto_link_saml_user - end - - def external_users_enabled? - !Gitlab::Saml::Config.external_groups.nil? - end - - def auth_hash=(auth_hash) - @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) - end - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 5a5ae7f19d4..1e45d074e0a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,15 +1,17 @@ module Gitlab class SearchResults class FoundBlob + include EncodingHelper + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id def initialize(opts = {}) @id = opts.fetch(:id, nil) - @filename = opts.fetch(:filename, nil) - @basename = opts.fetch(:basename, nil) + @filename = encode_utf8(opts.fetch(:filename, nil)) + @basename = encode_utf8(opts.fetch(:basename, nil)) @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) - @data = opts.fetch(:data, nil) + @data = encode_utf8(opts.fetch(:data, nil)) @per_page = opts.fetch(:per_page, 20) @project_id = opts.fetch(:project_id, nil) end @@ -60,22 +62,6 @@ module Gitlab without_count ? collection.without_count : collection end - def projects_count - @projects_count ||= projects.count - end - - def issues_count - @issues_count ||= issues.count - end - - def merge_requests_count - @merge_requests_count ||= merge_requests.count - end - - def milestones_count - @milestones_count ||= milestones.count - end - def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4ba44e0feef..3a8f5826818 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -69,13 +69,14 @@ module Gitlab # name - project disk path # # Ex. - # add_repository("/path/to/storage", "gitlab/gitlab-ci") + # create_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(storage, name) + def create_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') - gitaly_migrate(:create_repository) do |is_enabled| + gitaly_migrate(:create_repository, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled repository = Gitlab::Git::Repository.new(storage, relative_path, '') repository.gitaly_repository_client.create_repository @@ -85,7 +86,7 @@ module Gitlab Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) end end - rescue => err + rescue => err # Once the Rugged codes gets removes this can be improved Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false end @@ -125,13 +126,13 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) + def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -428,8 +429,8 @@ module Gitlab ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false) - vars = { force: forced, tags: !no_tags } + def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? @@ -487,8 +488,8 @@ module Gitlab Gitlab.config.gitlab_shell.git_timeout end - def gitaly_migrate(method, &block) - Gitlab::GitalyClient.migrate(method, &block) + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound, GRPC::BadStatus => e # Old Popen code returns [Error, output] to the caller, so we # need to do the same here... diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index b89ae2505c9..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - def call(worker, job, queue) - yield - - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - return unless MUTEX.try_lock - - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - Process.kill(signal, pid) - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb new file mode 100644 index 00000000000..c2b8d6de66e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -0,0 +1,133 @@ +require 'mutex_m' + +module Gitlab + module SidekiqMiddleware + class Shutdown + extend Mutex_m + + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # This exception can be used to request that the middleware start shutting down Sidekiq + WantShutdown = Class.new(StandardError) + + ShutdownWithoutRaise = Class.new(WantShutdown) + private_constant :ShutdownWithoutRaise + + # For testing only, to avoid race conditions (?) in Rspec mocks. + attr_reader :trace + + # We store the shutdown thread in a class variable to ensure that there + # can be only one shutdown thread in the process. + def self.create_shutdown_thread + mu_synchronize do + return unless @shutdown_thread.nil? + + @shutdown_thread = Thread.new { yield } + end + end + + # For testing only: so we can wait for the shutdown thread to finish. + def self.shutdown_thread + mu_synchronize { @shutdown_thread } + end + + # For testing only: so that we can reset the global state before each test. + def self.clear_shutdown_thread + mu_synchronize { @shutdown_thread = nil } + end + + def initialize + @trace = Queue.new if Rails.env.test? + end + + def call(worker, job, queue) + shutdown_exception = nil + + begin + yield + check_rss! + rescue WantShutdown => ex + shutdown_exception = ex + end + + return unless shutdown_exception + + self.class.create_shutdown_thread do + do_shutdown(worker, job, shutdown_exception) + end + + raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) + end + + private + + def do_shutdown(worker, job, shutdown_exception) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ + "#{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + + def check_rss! + return unless MAX_RSS > 0 + + current_rss = get_rss + return unless current_rss > MAX_RSS + + raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + kill(signal, pid) + end + + def pid + Process.pid + end + + def sleep(time) + if Rails.env.test? + @trace << [:sleep, time] + else + Kernel.sleep(time) + end + end + + def kill(signal, pid) + if Rails.env.test? + @trace << [:kill, signal, pid] + else + Process.kill(signal, pid) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index cc3c9a50555..466554e398c 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -31,10 +31,11 @@ module Gitlab raise NotImplementedError end - attr_accessor :project, :current_user, :params + attr_accessor :project, :current_user, :params, :chat_name - def initialize(project, user, params = {}) - @project, @current_user, @params = project, user, params.dup + def initialize(project, chat_name, params = {}) + @project, @current_user, @params = project, chat_name.user, params.dup + @chat_name = chat_name end private diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index a78408b0519..bb778f37096 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -5,6 +5,7 @@ module Gitlab Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::IssueMove, Gitlab::SlashCommands::Deploy ].freeze @@ -13,12 +14,13 @@ module Gitlab if command if command.allowed?(project, current_user) - command.new(project, current_user, params).execute(match) + command.new(project, chat_name, params).execute(match) else Gitlab::SlashCommands::Presenters::Access.new.access_denied end else - Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + Gitlab::SlashCommands::Help.new(project, chat_name, params) + .execute(available_commands, params[:text]) end end diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb new file mode 100644 index 00000000000..3985e635983 --- /dev/null +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -0,0 +1,45 @@ +module Gitlab + module SlashCommands + class IssueMove < IssueCommand + def self.match(text) + %r{ + \A # the beginning of a string + issue\s+move\s+ # the command + \#?(?<iid>\d+)\s+ # the issue id, may preceded by hash sign + (to\s+)? # aid the command to be much more human-ly + (?<project_path>[^\s]+) # named group for id of dest. project + }x.match(text) + end + + def self.help_message + 'issue move <issue_id> (to)? <project_path>' + end + + def self.allowed?(project, user) + can?(user, :admin_issue, project) + end + + def execute(match) + old_issue = find_by_iid(match[:iid]) + target_project = Project.find_by_full_path(match[:project_path]) + + unless current_user.can?(:read_project, target_project) && old_issue + return Gitlab::SlashCommands::Presenters::Access.new.not_found + end + + new_issue = Issues::MoveService.new(project, current_user) + .execute(old_issue, target_project) + + presenter(new_issue).present(old_issue) + rescue Issues::MoveService::MoveError => e + presenter(old_issue).display_move_error(e.message) + end + + private + + def presenter(issue) + Gitlab::SlashCommands::Presenters::IssueMove.new(issue) + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/issue_move.rb b/lib/gitlab/slash_commands/presenters/issue_move.rb new file mode 100644 index 00000000000..03921729941 --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/issue_move.rb @@ -0,0 +1,53 @@ +# coding: utf-8 +module Gitlab + module SlashCommands + module Presenters + class IssueMove < Presenters::Base + include Presenters::IssueBase + + def present(old_issue) + in_channel_response(moved_issue(old_issue)) + end + + def display_move_error(error) + message = header_with_list("The action was not successful, because:", [error]) + + ephemeral_response(text: message) + end + + private + + def moved_issue(old_issue) + { + attachments: [ + { + title: "#{@resource.title} ยท #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "Issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext(old_issue), + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :text, + :fields + ] + } + ] + } + end + + def pretext(old_issue) + "Moved issue *#{issue_link(old_issue)}* to *#{issue_link(@resource)}*" + end + + def issue_link(issue) + "[#{issue.to_reference}](#{project_issue_url(issue.project, issue)})" + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 86490a39cc1..5964bfe9960 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -38,7 +38,7 @@ module Gitlab end def project_link - "[#{project.name_with_namespace}](#{project.web_url})" + "[#{project.full_name}](#{project.web_url})" end def author_profile_link diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index c99316df667..562f15f403c 100644 --- a/lib/gitlab/slash_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -53,7 +53,7 @@ module Gitlab end def pretext - "Issue *#{@resource.to_reference}* from #{project.name_with_namespace}" + "Issue *#{@resource.to_reference}* from #{project.full_name}" end end end diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 7021b4b01b2..3669dedf0fe 100644 --- a/lib/gitlab/slash_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,4 +1,4 @@ -module Gitlab +module Gitlab # rubocop:disable Naming/FileName module SlashCommands Result = Struct.new(:type, :message) end diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb new file mode 100644 index 00000000000..9a2219b7d77 --- /dev/null +++ b/lib/gitlab/string_placeholder_replacer.rb @@ -0,0 +1,27 @@ +module Gitlab + class StringPlaceholderReplacer + # This method accepts the following paras + # - string: the string to be analyzed + # - placeholder_regex: i.e. /%{project_path|project_id|default_branch|commit_sha}/ + # - block: this block will be called with each placeholder found in the string using + # the placeholder regex. If the result of the block is nil, the original + # placeholder will be returned. + + def self.replace_string_placeholders(string, placeholder_regex = nil, &block) + return string if string.blank? || placeholder_regex.blank? || !block_given? + + replace_placeholders(string, placeholder_regex, &block) + end + + class << self + private + + # If the result of the block is nil, then the placeholder is returned + def replace_placeholders(string, placeholder_regex, &block) + string.gsub(/%{(#{placeholder_regex})}/) do |arg| + yield($~[1]) || arg + end + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index f9faa134206..c6ad997a4d4 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -14,7 +14,7 @@ module Gitlab end def mark(marker_ranges) - return rich_line unless marker_ranges + return rich_line unless marker_ranges&.any? if html_escaped rich_marker_ranges = [] diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index 7ebf1c0428c..b19aa6dea35 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,13 +1,15 @@ module Gitlab class StringRegexMarker < StringRangeMarker def mark(regex, group: 0, &block) - regex_match = raw_line.match(regex) - return rich_line unless regex_match + ranges = [] - begin_index, end_index = regex_match.offset(group) - name_range = begin_index..(end_index - 1) + raw_line.scan(regex) do + begin_index, end_index = Regexp.last_match.offset(group) - super([name_range], &block) + ranges << (begin_index..(end_index - 1)) + end + + super(ranges, &block) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 9d13d1d781f..37d3512990e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,6 +9,7 @@ module Gitlab license_usage_data.merge(system_usage_data) .merge(features_usage_data) .merge(components_usage_data) + .merge(cycle_analytics_usage_data) end def to_json(force_refresh: false) @@ -71,6 +72,10 @@ module Gitlab } end + def cycle_analytics_usage_data + Gitlab::CycleAnalytics::UsageData.new.to_json + end + def features_usage_data features_usage_data_ce end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index ff4dc29efea..24393f96d96 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -31,7 +31,7 @@ module Gitlab return false unless can_access_git? if user.requires_ldap_check? && user.try_obtain_ldap_lease - return false unless Gitlab::LDAP::Access.allowed?(user) + return false unless Gitlab::Auth::LDAP::Access.allowed?(user) end true @@ -63,13 +63,12 @@ module Gitlab request_cache def can_push_to_branch?(ref) return false unless can_access_git? + return false unless user.can?(:push_code, project) || project.branch_allows_maintainer_push?(user, ref) if protected?(ProtectedBranch, project, ref) - return true if project.user_can_push_to_empty_repo?(user) - - protected_branch_accessible_to?(ref, action: :push) + project.user_can_push_to_empty_repo?(user) || protected_branch_accessible_to?(ref, action: :push) else - user.can?(:push_code, project) + true end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index fa22f0e37b2..dc9391f32cf 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -67,5 +67,13 @@ module Gitlab nil end + + # Used in EE + # Accepts either an Array or a String and returns an array + def ensure_array_from_string(string_or_array) + return string_or_array if string_or_array.is_a?(Array) + + string_or_array.split(',').map(&:strip) + end end end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb new file mode 100644 index 00000000000..1ef369a4b67 --- /dev/null +++ b/lib/gitlab/verify/batch_verifier.rb @@ -0,0 +1,64 @@ +module Gitlab + module Verify + class BatchVerifier + attr_reader :batch_size, :start, :finish + + def initialize(batch_size:, start: nil, finish: nil) + @batch_size = batch_size + @start = start + @finish = finish + end + + # Yields a Range of IDs and a Hash of failed verifications (object => error) + def run_batches(&blk) + relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches + range = relation.first.id..relation.last.id + failures = run_batch(relation) + + yield(range, failures) + end + end + + def name + raise NotImplementedError.new + end + + def describe(_object) + raise NotImplementedError.new + end + + private + + def run_batch(relation) + relation.map { |upload| verify(upload) }.compact.to_h + end + + def verify(object) + expected = expected_checksum(object) + actual = actual_checksum(object) + + raise 'Checksum missing' unless expected.present? + raise 'Checksum mismatch' unless expected == actual + + nil + rescue => err + [object, err] + end + + # This should return an ActiveRecord::Relation suitable for calling #in_batches on + def relation + raise NotImplementedError.new + end + + # The checksum we expect the object to have + def expected_checksum(_object) + raise NotImplementedError.new + end + + # The freshly-recalculated checksum of the object + def actual_checksum(_object) + raise NotImplementedError.new + end + end + end +end diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb new file mode 100644 index 00000000000..03500a61074 --- /dev/null +++ b/lib/gitlab/verify/job_artifacts.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class JobArtifacts < BatchVerifier + def name + 'Job artifacts' + end + + def describe(object) + "Job artifact: #{object.id}" + end + + private + + def relation + ::Ci::JobArtifact.all + end + + def expected_checksum(artifact) + artifact.file_sha256 + end + + def actual_checksum(artifact) + Digest::SHA256.file(artifact.file.path).hexdigest + end + end + end +end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb new file mode 100644 index 00000000000..fe51edbdeeb --- /dev/null +++ b/lib/gitlab/verify/lfs_objects.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class LfsObjects < BatchVerifier + def name + 'LFS objects' + end + + def describe(object) + "LFS object: #{object.oid}" + end + + private + + def relation + LfsObject.all + end + + def expected_checksum(lfs_object) + lfs_object.oid + end + + def actual_checksum(lfs_object) + LfsObject.calculate_oid(lfs_object.file.path) + end + end + end +end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb new file mode 100644 index 00000000000..dd138e6b92b --- /dev/null +++ b/lib/gitlab/verify/rake_task.rb @@ -0,0 +1,53 @@ +module Gitlab + module Verify + class RakeTask + def self.run!(verify_kls) + verifier = verify_kls.new( + batch_size: ENV.fetch('BATCH', 200).to_i, + start: ENV['ID_FROM'], + finish: ENV['ID_TO'] + ) + + verbose = Gitlab::Utils.to_boolean(ENV['VERBOSE']) + + new(verifier, verbose).run! + end + + attr_reader :verifier, :output + + def initialize(verifier, verbose) + @verifier = verifier + @verbose = verbose + end + + def run! + say "Checking integrity of #{verifier.name}" + + verifier.run_batches { |*args| run_batch(*args) } + + say 'Done!' + end + + def verbose? + !!@verbose + end + + private + + def say(text) + puts(text) # rubocop:disable Rails/Output + end + + def run_batch(range, failures) + status_color = failures.empty? ? :green : :red + say "- #{range}: Failures: #{failures.count}".color(status_color) + + return unless verbose? + + failures.each do |object, error| + say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + end + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb new file mode 100644 index 00000000000..6972e517ea5 --- /dev/null +++ b/lib/gitlab/verify/uploads.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class Uploads < BatchVerifier + def name + 'Uploads' + end + + def describe(object) + "Upload: #{object.id}" + end + + private + + def relation + Upload.all + end + + def expected_checksum(upload) + upload.checksum + end + + def actual_checksum(upload) + Upload.hexdigest(upload.absolute_path) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 823df67ea39..0b0d667d4fd 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -10,6 +10,7 @@ module Gitlab INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze + ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 # bytes https://tools.ietf.org/html/rfc4868#section-2.6 @@ -17,6 +18,8 @@ module Gitlab class << self def git_http_ok(repository, is_wiki, user, action, show_all_refs: false) + raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) + project = repository.project repo_path = repository.path_to_repo params = { @@ -31,24 +34,7 @@ module Gitlab token: Gitlab::GitalyClient.token(project.repository_storage) } params[:Repository] = repository.gitaly_repository.to_h - - feature_enabled = case action.to_s - when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?( - :post_receive_pack, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT - ) - when 'git_upload_pack' - true - when 'info_refs' - true - else - raise "Unsupported action: #{action}" - end - - if feature_enabled - params[:GitalyServer] = server - end + params[:GitalyServer] = server params end diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb index 99a82c849e0..1aeaa387a49 100644 --- a/lib/google_api/auth.rb +++ b/lib/google_api/auth.rb @@ -32,7 +32,7 @@ module GoogleApi private def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "google_oauth2" } + Gitlab::Auth::OAuth::Provider.config_for('google_oauth2') end def client diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index f5485eb89fa..adbed20f152 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -1,4 +1,4 @@ -unless Rails.env.production? +unless Rails.env.production? # rubocop:disable Naming/FileName require 'haml_lint/haml_visitor' require 'haml_lint/linter' require 'haml_lint/linter_registry' @@ -12,6 +12,12 @@ unless Rails.env.production? record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') end + + def visit_tag(node) + return unless node.tag_name == 'script' + + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') + end end end end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index ef08bd46e17..65ccdb3c347 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -83,6 +83,12 @@ module Mattermost end end + def delete(path, options = {}) + handle_exceptions do + self.class.delete(path, options.merge(headers: @headers)) + end + end + private def create diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index b2511f3af1d..75513a9ba04 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -16,10 +16,9 @@ module Mattermost end # The deletion is done async, so the response is fast. - # On the mattermost side, this triggers an soft deletion first, after which - # the actuall data is removed + # On the mattermost side, this triggers an soft deletion def destroy(team_id:) - session_delete("/api/v4/teams/#{team_id}?permanent=true") + session_delete("/api/v4/teams/#{team_id}") end end end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index d519d8e86fa..ab35f7a2258 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -10,11 +10,29 @@ module Peek end def results - { duration: formatted_duration, calls: calls } + { + duration: formatted_duration, + calls: calls, + details: details + } end private + def details + ::Gitlab::GitalyClient.list_call_details + .values + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def format_call_details(call) + pretty_request = call[:request]&.reject { |k, v| v.blank? }.to_h.pretty_inspect + + call.merge(duration: (call[:duration] * 1000).round(3), + request: pretty_request || {}) + end + def formatted_duration ms = duration * 1000 if ms >= 1000 diff --git a/lib/repository_cache.rb b/lib/repository_cache.rb deleted file mode 100644 index 068a95790c0..00000000000 --- a/lib/repository_cache.rb +++ /dev/null @@ -1,30 +0,0 @@ -# Interface to the Redis-backed cache store used by the Repository model -class RepositoryCache - attr_reader :namespace, :backend, :project_id - - def initialize(namespace, project_id, backend = Rails.cache) - @namespace = namespace - @backend = backend - @project_id = project_id - end - - def cache_key(type) - "#{type}:#{namespace}:#{project_id}" - end - - def expire(key) - backend.delete(cache_key(key)) - end - - def fetch(key, &block) - backend.fetch(cache_key(key), &block) - end - - def exist?(key) - backend.exist?(cache_key(key)) - end - - def read(key) - backend.read(cache_key(key)) - end -end diff --git a/lib/rouge/plugins/common_mark.rb b/lib/rouge/plugins/common_mark.rb new file mode 100644 index 00000000000..8f9de061124 --- /dev/null +++ b/lib/rouge/plugins/common_mark.rb @@ -0,0 +1,20 @@ +# A rouge plugin for CommonMark markdown engine. +# Used to highlight code generated by CommonMark. + +module Rouge + module Plugins + module CommonMark + def code_block(code, language) + lexer = Lexer.find_fancy(language, code) || Lexers::PlainText + + formatter = rouge_formatter(lexer) + formatter.format(lexer.lex(code)) + end + + # override this method for custom formatting behavior + def rouge_formatter(lexer) + Formatters::HTMLLegacy.new(css_class: "highlight #{lexer.tag}") + end + end + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index 914ed794601..6227e461d24 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -50,7 +50,7 @@ module SystemCheck if should_sanitize? "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " else - "#{project.name_with_namespace.color(:yellow)} ... " + "#{project.full_name.color(:yellow)} ... " end end diff --git a/lib/tasks/gitlab/artifacts/check.rake b/lib/tasks/gitlab/artifacts/check.rake new file mode 100644 index 00000000000..a105261ed51 --- /dev/null +++ b/lib/tasks/gitlab/artifacts/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :artifacts do + desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::JobArtifacts) + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index e05a3aad824..2403f57f05a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -336,7 +336,7 @@ namespace :gitlab do warn_user_is_not_gitlab start_checking "LDAP" - if Gitlab::LDAP::Config.enabled? + if Gitlab::Auth::LDAP::Config.enabled? check_ldap(args.limit) else puts 'LDAP is disabled in config/gitlab.yml' @@ -346,13 +346,13 @@ namespace :gitlab do end def check_ldap(limit) - servers = Gitlab::LDAP::Config.providers + servers = Gitlab::Auth::LDAP::Config.providers servers.each do |server| puts "Server: #{server}" begin - Gitlab::LDAP::Adapter.open(server) do |adapter| + Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| check_ldap_auth(adapter) puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 5a53eac0897..2453079911d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -87,7 +87,7 @@ namespace :gitlab do print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." - if Gitlab::LDAP::Access.allowed?(user) + if Gitlab::Auth::LDAP::Access.allowed?(user) puts " [OK]".color(:green) else if block_flag diff --git a/lib/tasks/gitlab/exclusive_lease.rake b/lib/tasks/gitlab/exclusive_lease.rake new file mode 100644 index 00000000000..83722bf6d94 --- /dev/null +++ b/lib/tasks/gitlab/exclusive_lease.rake @@ -0,0 +1,9 @@ +namespace :gitlab do + namespace :exclusive_lease do + desc 'GitLab | Clear existing exclusive leases for specified scope (default: *)' + task :clear, [:scope] => [:environment] do |_, args| + args[:scope].nil? ? Gitlab::ExclusiveLease.reset_all! : Gitlab::ExclusiveLease.reset_all!(args[:scope]) + puts 'All exclusive lease entries were removed.' + end + end +end diff --git a/lib/tasks/gitlab/lfs/check.rake b/lib/tasks/gitlab/lfs/check.rake new file mode 100644 index 00000000000..869463d4e5d --- /dev/null +++ b/lib/tasks/gitlab/lfs/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :lfs do + desc 'GitLab | LFS | Check integrity of uploaded LFS objects' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::LfsObjects) + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 844664b12d4..4fcbbbf8c9d 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -69,7 +69,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage, + if Gitlab::Shell.new.create_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake new file mode 100644 index 00000000000..fd2a4f2d11a --- /dev/null +++ b/lib/tasks/gitlab/traces.rake @@ -0,0 +1,24 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Archive legacy traces to trace artifacts" +namespace :gitlab do + namespace :traces do + task archive: :environment do + logger = Logger.new(STDOUT) + logger.info('Archiving legacy traces') + + Ci::Build.finished + .where('NOT EXISTS (?)', + Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + .order(id: :asc) + .find_in_batches(batch_size: 1000) do |jobs| + job_ids = jobs.map { |job| [job.id] } + + ArchiveTraceWorker.bulk_perform_async(job_ids) + + logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") + end + end + end +end diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake deleted file mode 100644 index df31567ce64..00000000000 --- a/lib/tasks/gitlab/uploads.rake +++ /dev/null @@ -1,44 +0,0 @@ -namespace :gitlab do - namespace :uploads do - desc 'GitLab | Uploads | Check integrity of uploaded files' - task check: :environment do - puts 'Checking integrity of uploaded files' - - uploads_batches do |batch| - batch.each do |upload| - puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) - - if upload.exist? - check_checksum(upload) - else - puts " * File does not exist on the file system".color(:red) - end - end - end - - puts 'Done!' - end - - def batch_size - ENV.fetch('BATCH', 200).to_i - end - - def calculate_checksum(absolute_path) - Digest::SHA256.file(absolute_path).hexdigest - end - - def check_checksum(upload) - checksum = calculate_checksum(upload.absolute_path) - - if checksum != upload.checksum - puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) - end - end - - def uploads_batches(&block) - Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches - yield relation - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/check.rake b/lib/tasks/gitlab/uploads/check.rake new file mode 100644 index 00000000000..2be2ec7f9c9 --- /dev/null +++ b/lib/tasks/gitlab/uploads/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::Uploads) + end + end +end diff --git a/lib/tasks/plugins.rake b/lib/tasks/plugins.rake new file mode 100644 index 00000000000..e73dd7e68df --- /dev/null +++ b/lib/tasks/plugins.rake @@ -0,0 +1,16 @@ +namespace :plugins do + desc 'Validate existing plugins' + task validate: :environment do + puts 'Validating plugins from /plugins directory' + + Gitlab::Plugin.files.each do |file| + success, message = Gitlab::Plugin.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) + + if success + puts "* #{file} succeed (zero exit code)." + else + puts "* #{file} failure (non-zero exit code). #{message}" + end + end + end +end |