diff options
author | Pawel Chojnacki <pawel@chojnacki.ws> | 2018-03-06 16:08:11 +0100 |
---|---|---|
committer | Pawel Chojnacki <pawel@chojnacki.ws> | 2018-03-06 16:08:11 +0100 |
commit | 4ff8db0d2e8371dfdae2ddef8a8595c1ef80c3d4 (patch) | |
tree | 3ff849c51603151baaf85ee0436d9c2f3e47915d /lib | |
parent | 60f5250893626620920d2f108df3b3e98d974f0e (diff) | |
parent | 5e8138aa54492dd3ace42d889ba01f82e8e19c83 (diff) | |
download | gitlab-ce-4ff8db0d2e8371dfdae2ddef8a8595c1ef80c3d4.tar.gz |
Merge remote-tracking branch 'upstream/master' into 5029-support-cluster-metrics-ce5029-support-cluster-metrics-ce
# Conflicts:
# spec/models/clusters/applications/prometheus_spec.rb
Diffstat (limited to 'lib')
127 files changed, 3820 insertions, 2302 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 60ae5e6b9a2..ae13c248171 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -53,7 +53,10 @@ module API put ':id/access_requests/:user_id/approve' do source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute + access_requester = source.requesters.find_by!(user_id: params[:user_id]) + member = ::Members::ApproveAccessRequestService + .new(current_user, declared_params) + .execute(access_requester) status :created present member, with: Entities::Member @@ -70,8 +73,7 @@ module API member = source.requesters.find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do - ::Members::DestroyService.new(source, current_user, params) - .execute(:requesters) + ::Members::DestroyService.new(current_user).execute(member) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 754549f72f0..b1b247b70b9 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -108,6 +108,7 @@ module API mount ::API::AccessRequests mount ::API::Applications mount ::API::AwardEmoji + mount ::API::Badges mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages diff --git a/lib/api/badges.rb b/lib/api/badges.rb new file mode 100644 index 00000000000..334948b2995 --- /dev/null +++ b/lib/api/badges.rb @@ -0,0 +1,134 @@ +module API + class Badges < Grape::API + include PaginationParams + + before { authenticate_non_get! } + + helpers ::API::Helpers::BadgesHelpers + + helpers do + def find_source_if_admin(source_type) + source = find_source(source_type, params[:id]) + + authorize_admin_source!(source_type, source) + + source + end + end + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The ID of a #{source_type}" + end + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Gets a list of #{source_type} badges viewable by the authenticated user." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + use :pagination + end + get ":id/badges" do + source = find_source(source_type, params[:id]) + + present_badges(source, paginate(source.badges)) + end + + desc "Preview a badge from a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::BasicBadgeDetails + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + get ":id/badges/render" do + authenticate! + + source = find_source_if_admin(source_type) + + badge = ::Badges::BuildService.new(declared_params(include_missing: false)) + .execute(source) + + if badge.valid? + present_badges(source, badge, with: Entities::BasicBadgeDetails) + else + render_validation_error!(badge) + end + end + + desc "Gets a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + get ":id/badges/:badge_id" do + source = find_source(source_type, params[:id]) + badge = find_badge(source) + + present_badges(source, badge) + end + + desc "Adds a badge to a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + post ":id/badges" do + source = find_source_if_admin(source_type) + + badge = ::Badges::CreateService.new(declared_params(include_missing: false)).execute(source) + + if badge.persisted? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc "Updates a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + optional :link_url, type: String, desc: 'URL of the badge link' + optional :image_url, type: String, desc: 'URL of the badge image' + end + put ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + + badge = ::Badges::UpdateService.new(declared_params(include_missing: false)) + .execute(find_badge(source)) + + if badge.valid? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc 'Removes a badge from a project or group.' do + detail 'This feature was introduced in GitLab 10.6.' + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + delete ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + badge = find_badge(source) + + if badge.is_a?(GroupBadge) && source.is_a?(Project) + error!('To delete a Group badge please use the Group endpoint', 403) + end + + destroy_conditionally!(badge) + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 1794207e29b..13cfba728fa 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -16,6 +16,10 @@ module API render_api_error!('The branch refname is invalid', 400) end end + + params :filter_params do + optional :search, type: String, desc: 'Return list of branches matching the search criteria' + end end params do @@ -27,15 +31,23 @@ module API end params do use :pagination + use :filter_params end get ':id/repository/branches' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') repository = user_project.repository - branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) + + branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names + present( + paginate(::Kaminari.paginate_array(branches)), + with: Entities::Branch, + project: user_project, + merged_branch_names: merged_branch_names + ) end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 3d6e78d2d80..982f45425a3 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -18,25 +18,28 @@ module API optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' optional :path, type: String, desc: 'The file path' + optional :all, type: Boolean, desc: 'Every commit will be returned' use :pagination end get ':id/repository/commits' do path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] + all = params[:all] commits = user_project.repository.commits(ref, path: path, limit: params[:per_page], offset: offset, before: before, - after: after) + after: after, + all: all) commit_count = - if path || before || after - user_project.repository.count_commits(ref: ref, path: path, before: before, after: after) + if all || path || before || after + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all) else # Cacheable commit count. user_project.repository.commit_count_for_ref(ref) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 167878ba600..e5bcbface6b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -71,7 +71,7 @@ module API end class ProjectHook < Hook - expose :project_id, :issues_events + expose :project_id, :issues_events, :confidential_issues_events expose :note_events, :pipeline_events, :wiki_page_events expose :job_events end @@ -481,6 +481,10 @@ module API expose :id end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class MergeRequestSimple < ProjectEntity expose :title expose :web_url do |merge_request, options| @@ -546,6 +550,42 @@ module API expose :changes_count do |merge_request, _options| merge_request.merge_request_diff.real_size end + + expose :merged_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.merged_by + end + + expose :merged_at do |merge_request, _options| + merge_request.metrics&.merged_at + end + + expose :closed_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.latest_closed_by + end + + expose :closed_at do |merge_request, _options| + merge_request.metrics&.latest_closed_at + end + + expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.latest_build_started_at + end + + expose :latest_build_finished_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.latest_build_finished_at + end + + expose :first_deployed_to_production_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.first_deployed_to_production_at + end + + expose :pipeline, using: Entities::PipelineBasic, if: -> (_, options) { build_available?(options) } do |merge_request, _options| + merge_request.metrics&.pipeline + end + + def build_available?(options) + options[:project]&.feature_available?(:builds, options[:current_user]) + end end class MergeRequestChanges < MergeRequest @@ -909,10 +949,6 @@ module API expose :filename, :size end - class PipelineBasic < Grape::Entity - expose :id, :sha, :ref, :status - end - class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -1199,5 +1235,23 @@ module API expose :startline expose :project_id end + + class BasicBadgeDetails < Grape::Entity + expose :link_url + expose :image_url + expose :rendered_link_url do |badge, options| + badge.rendered_link_url(options.fetch(:project, nil)) + end + expose :rendered_image_url do |badge, options| + badge.rendered_image_url(options.fetch(:project, nil)) + end + end + + class Badge < BasicBadgeDetails + expose :id + expose :kind do |badge| + badge.type == 'ProjectBadge' ? 'project' : 'group' + end + end end end diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb new file mode 100644 index 00000000000..1f8afbf3c90 --- /dev/null +++ b/lib/api/helpers/badges_helpers.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module BadgesHelpers + include ::API::Helpers::MembersHelpers + + def find_badge(source) + source.badges.find(params[:badge_id]) + end + + def present_badges(source, records, options = {}) + entity_type = options[:with] || Entities::Badge + badge_params = badge_source_params(source).merge(with: entity_type) + + present records, badge_params + end + + def badge_source_params(source) + project = if source.is_a?(Project) + source + else + GroupProjectsFinder.new(group: source, current_user: current_user).execute.first + end + + { project: project } + end + end + end +end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index fbe30192a16..35ac0b4cbca 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -9,16 +9,22 @@ module API Gitlab::CurrentSettings.runners_registration_token) end - def get_runner_version_from_params - return unless params['info'].present? + def authenticate_runner! + forbidden! unless current_runner - attributes_for_keys(%w(name version revision platform architecture), params['info']) + current_runner + .update_cached_info(get_runner_details_from_request) end - def authenticate_runner! - forbidden! unless current_runner + def get_runner_details_from_request + return get_runner_ip unless params['info'].present? + + attributes_for_keys(%w(name version revision platform architecture), params['info']) + .merge(get_runner_ip) + end - current_runner.update_cached_info(get_runner_version_from_params) + def get_runner_ip + { ip_address: request.ip } end def current_runner diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b6c278c89d0..f74b3b26802 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -32,6 +32,8 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], diff --git a/lib/api/members.rb b/lib/api/members.rb index bc1de37284a..8b12986d09e 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -81,12 +81,16 @@ module API source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) - member = source.members.find_by!(user_id: params.delete(:user_id)) + member = source.members.find_by!(user_id: params[:user_id]) + updated_member = + ::Members::UpdateService + .new(current_user, declared_params(include_missing: false)) + .execute(member) - if member.update_attributes(declared_params(include_missing: false)) - present member, with: Entities::Member + if updated_member.valid? + present updated_member, with: Entities::Member else - render_validation_error!(member) + render_validation_error!(updated_member) end end @@ -99,7 +103,7 @@ module API member = source.members.find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do - ::Members::DestroyService.new(source, current_user, declared_params).execute + ::Members::DestroyService.new(current_user).execute(member) end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 719afa09295..8c02972b421 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -42,12 +42,16 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' + optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' optional :search, type: String, desc: 'Search merge requests for text present in the title or description' use :pagination end @@ -220,7 +224,7 @@ module API get ':id/merge_requests/:merge_request_iid/changes' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequestChanges, current_user: current_user + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user, project: user_project end desc 'Get the merge request pipelines' do diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 86066e2b58f..f82241058e5 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -10,6 +10,7 @@ module API requires :url, type: String, desc: "The URL to send the request to" optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 5469cba69a6..7e6c33ec33d 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -16,7 +16,8 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) end post '/' do - attributes = attributes_for_keys [:description, :locked, :run_untagged, :tag_list] + attributes = attributes_for_keys([:description, :locked, :run_untagged, :tag_list]) + .merge(get_runner_details_from_request) runner = if runner_registration_token_valid? @@ -30,7 +31,6 @@ module API return forbidden! unless runner if runner.id - runner.update(get_runner_version_from_params) present runner, with: Entities::RunnerRegistrationDetails else not_found! @@ -204,6 +204,7 @@ module API optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) end @@ -224,7 +225,7 @@ module API expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata job.artifacts_expire_in = expire_in diff --git a/lib/api/services.rb b/lib/api/services.rb index 51e33e2c686..6c97659166d 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,6 +1,7 @@ +# frozen_string_literal: true module API class Services < Grape::API - chat_notification_settings = [ + CHAT_NOTIFICATION_SETTINGS = [ { required: true, name: :webhook, @@ -19,9 +20,9 @@ module API type: String, desc: 'The default chat channel' } - ] + ].freeze - chat_notification_flags = [ + CHAT_NOTIFICATION_FLAGS = [ { required: false, name: :notify_only_broken_pipelines, @@ -34,9 +35,9 @@ module API type: Boolean, desc: 'Send notifications only for the default branch' } - ] + ].freeze - chat_notification_channels = [ + CHAT_NOTIFICATION_CHANNELS = [ { required: false, name: :push_channel, @@ -85,9 +86,9 @@ module API type: String, desc: 'The name of the channel to receive wiki_page_events notifications' } - ] + ].freeze - chat_notification_events = [ + CHAT_NOTIFICATION_EVENTS = [ { required: false, name: :push_events, @@ -136,7 +137,7 @@ module API type: Boolean, desc: 'Enable notifications for wiki_page_events' } - ] + ].freeze services = { 'asana' => [ @@ -627,10 +628,10 @@ module API } ], 'slack' => [ - chat_notification_settings, - chat_notification_flags, - chat_notification_channels, - chat_notification_events + CHAT_NOTIFICATION_SETTINGS, + CHAT_NOTIFICATION_FLAGS, + CHAT_NOTIFICATION_CHANNELS, + CHAT_NOTIFICATION_EVENTS ].flatten, 'microsoft-teams' => [ { @@ -641,10 +642,10 @@ module API } ], 'mattermost' => [ - chat_notification_settings, - chat_notification_flags, - chat_notification_channels, - chat_notification_events + CHAT_NOTIFICATION_SETTINGS, + CHAT_NOTIFICATION_FLAGS, + CHAT_NOTIFICATION_CHANNELS, + CHAT_NOTIFICATION_EVENTS ].flatten, 'teamcity' => [ { @@ -724,7 +725,22 @@ module API ] end - trigger_services = { + SERVICES = services.freeze + SERVICE_CLASSES = service_classes.freeze + + SERVICE_CLASSES.each do |service| + event_names = service.try(:event_names) || next + event_names.each do |event_name| + SERVICES[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end + end + + TRIGGER_SERVICES = { 'mattermost-slash-commands' => [ { name: :token, @@ -756,22 +772,9 @@ module API end end - services.each do |service_slug, settings| + SERVICES.each do |service_slug, settings| desc "Set #{service_slug} service for project" params do - service_classes.each do |service| - event_names = service.try(:event_names) || next - event_names.each do |event_name| - services[service.to_param.tr("_", "-")] << { - required: false, - name: event_name.to_sym, - type: String, - desc: ServicesHelper.service_event_description(event_name) - } - end - end - services.freeze - settings.each do |setting| if setting[:required] requires setting[:name], type: setting[:type], desc: setting[:desc] @@ -794,7 +797,7 @@ module API desc "Delete a service for project" params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' end delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) @@ -814,7 +817,7 @@ module API success Entities::ProjectService end params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) @@ -822,7 +825,7 @@ module API end end - trigger_services.each do |service_slug, settings| + TRIGGER_SERVICES.each do |service_slug, settings| helpers do def slash_command_service(project, service_slug, params) project.services.active.where(template: false).find do |service| diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 2ccbb9da1c5..68b4d7c3982 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -252,8 +252,9 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :pipeline_events + expose :push_events, :issues_events, :confidential_issues_events + expose :merge_requests_events, :tag_push_events, :note_events + expose :pipeline_events expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| @@ -262,8 +263,9 @@ module API end class ProjectHook < ::API::Entities::Hook - expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :pipeline_events, :wiki_page_events + expose :project_id, :issues_events, :confidential_issues_events + expose :merge_requests_events, :note_events, :pipeline_events + expose :wiki_page_events expose :job_events, as: :build_events end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index d7bde8ceb89..88dd598f1e9 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -124,7 +124,7 @@ module API status(200 ) { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(source, current_user, declared_params).execute + ::Members::DestroyService.new(current_user).execute(member) present member, with: ::API::Entities::Member end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb index 51014591a93..631944150c7 100644 --- a/lib/api/v3/project_hooks.rb +++ b/lib/api/v3/project_hooks.rb @@ -11,6 +11,7 @@ module API requires :url, type: String, desc: "The URL to send the request to" optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index e7e6a90b5fd..c9e3f8ce42b 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -174,7 +174,9 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, parent, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, + link_content: !!link_content, + link_reference: link_reference) url = if matches.names.include?("url") && matches[:url] @@ -194,12 +196,13 @@ module Banzai end end - def data_attributes_for(text, project, object, link: false) + def data_attributes_for(text, project, object, link_content: false, link_reference: false) data_attribute( - original: text, - link: link, - project: project.id, - object_sym => object.id + original: text, + link: link_content, + link_reference: link_reference, + project: project.id, + object_sym => object.id ) end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index b8d2673c1a6..75b64ae9af2 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -25,8 +25,8 @@ module Banzai # period or comma for punctuation without those characters being included # in the generated link. # - # Rubular: http://rubular.com/r/cxjPyZc7Sb - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://\S+)(?<!,|\.)} + # Rubular: http://rubular.com/r/JzPhi6DCZp + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -35,53 +35,19 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') - and not(starts-with(., 'http')) - and not(starts-with(., 'ftp')) ]).freeze + PUNCTUATION_PAIRS = { + "'" => "'", + '"' => '"', + ')' => '(', + ']' => '[', + '}' => '{' + }.freeze + def call return doc if context[:autolink] == false - rinku_parse - text_parse - end - - private - - # Run the text through Rinku as a first pass - # - # This will quickly autolink http(s) and ftp links. - # - # `@doc` will be re-parsed with the HTML String from Rinku. - def rinku_parse - # Convert the options from a Hash to a String that Rinku expects - options = tag_options(link_options) - - # NOTE: We don't parse email links because it will erroneously match - # external Commit and CommitRange references. - # - # The final argument tells Rinku to link short URLs that don't include a - # period (e.g., http://localhost:3000/) - rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) - - return if rinku == html - - # Rinku returns a String, so parse it back to a Nokogiri::XML::Document - # for further processing. - @doc = parse_html(rinku) - end - - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - - # Autolinks any text matching LINK_PATTERN that Rinku didn't already - # replace - def text_parse doc.xpath(TEXT_QUERY).each do |node| content = node.to_html @@ -97,6 +63,16 @@ module Banzai doc end + private + + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + def autolink_match(match) # start by stripping out dangerous links begin @@ -112,12 +88,30 @@ module Banzai match.gsub!(/((?:&[\w#]+;)+)\z/, '') dropped = ($1 || '').html_safe + # To match the behaviour of Rinku, if the matched link ends with a + # closing part of a matched pair of punctuation, we remove that trailing + # character unless there are an equal number of closing and opening + # characters in the link. + if match.end_with?(*PUNCTUATION_PAIRS.keys) + close_character = match[-1] + close_count = match.count(close_character) + open_character = PUNCTUATION_PAIRS[close_character] + open_count = match.count(open_character) + + if open_count != close_count || open_character == close_character + dropped += close_character + match = match[0..-2] + end + end + options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + content_tag(:a, match.html_safe, options) + dropped end def autolink_filter(text) - text.gsub(LINK_PATTERN) { |match| autolink_match(match) } + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + autolink_match(link) + end end def link_options diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 827df7c08ae..fd457bebf03 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -42,16 +42,33 @@ module Banzai next if visible.include?(node) doc_data[:visible_reference_count] -= 1 - # The reference should be replaced by the original link's content, - # which is not always the same as the rendered one. - content = node.attr('data-original') || node.inner_html - node.replace(content) + redacted_content = redacted_node_content(node) + node.replace(redacted_content) end end metadata end + # Return redacted content of given node as either the original link (<a> tag), + # the original content (text), or the inner HTML of the node. + # + def redacted_node_content(node) + original_content = node.attr('data-original') + link_reference = node.attr('data-link-reference') + + # Build the raw <a> tag just with a link as href and content if + # it's originally a link pattern. We shouldn't return a plain text href. + original_link = + if link_reference == 'true' && href = original_content + %(<a href="#{href}">#{href}</a>) + end + + # The reference should be replaced by the original link's content, + # which is not always the same as the rendered one. + original_link || original_content || node.inner_html + end + def redact_cross_project_references(documents) extractor = Banzai::IssuableExtractor.new(project, user) issuables = extractor.extract(documents) diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index b9279c33f5b..ba5a9e2f04c 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -57,7 +57,7 @@ module Bitbucket end def provider - Gitlab::OAuth::Provider.config_for('bitbucket') + Gitlab::Auth::OAuth::Provider.config_for('bitbucket') end def options diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 05932378173..f5ccf952cf9 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -40,8 +40,8 @@ module Gitlab end def find_with_user_password(login, password) - # Avoid resource intensive login checks if password is not provided - return unless password.present? + # Avoid resource intensive checks if login credentials are not provided + return unless login.present? && password.present? # Nothing to do here if internal auth is disabled and LDAP is # not configured @@ -50,14 +50,26 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - Gitlab::LDAP::Authentication.login(login, password) - elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user if user.active? && user.valid_password?(password) + return if user && !user.active? + + authenticators = [] + + if user + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, 'database') + + # Add authenticators for all identities if user is not nil + user&.identities&.each do |identity| + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, identity.provider) + end + else + # If no user is provided, try LDAP. + # LDAP users are only authenticated via LDAP + authenticators << Gitlab::Auth::LDAP::Authentication end + + authenticators.compact! + + user if authenticators.find { |auth| auth.login(login, password) } end end @@ -85,7 +97,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb new file mode 100644 index 00000000000..260a77058a4 --- /dev/null +++ b/lib/gitlab/auth/database/authentication.rb @@ -0,0 +1,16 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module Database + class Authentication < Gitlab::Auth::OAuth::Authentication + def login(login, password) + return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + + user&.valid_password?(password) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb new file mode 100644 index 00000000000..77c0ddc2d48 --- /dev/null +++ b/lib/gitlab/auth/ldap/access.rb @@ -0,0 +1,89 @@ +# LDAP authorization model +# +# * Check if we are allowed access (not blocked) +# +module Gitlab + module Auth + module LDAP + class Access + attr_reader :provider, :user + + def self.open(user, &block) + Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| + block.call(self.new(user, adapter)) + end + end + + def self.allowed?(user) + self.open(user) do |access| + if access.allowed? + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute + + true + else + false + end + end + end + + def initialize(user, adapter = nil) + @adapter = adapter + @user = user + @provider = user.ldap_identity.provider + end + + def allowed? + if ldap_user + unless ldap_config.active_directory + unblock_user(user, 'is available again') if user.ldap_blocked? + return true + end + + # Block user in GitLab if he/she was blocked in AD + if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) + block_user(user, 'is disabled in Active Directory') + false + else + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? + true + end + else + # Block the user if they no longer exist in LDAP/AD + block_user(user, 'does not exist anymore') + false + end + end + + def adapter + @adapter ||= Gitlab::Auth::LDAP::Adapter.new(provider) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def ldap_user + @ldap_user ||= Gitlab::Auth::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb new file mode 100644 index 00000000000..caf2d18c668 --- /dev/null +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -0,0 +1,110 @@ +module Gitlab + module Auth + module LDAP + class Adapter + attr_reader :provider, :ldap + + def self.open(provider, &block) + Net::LDAP.open(config(provider).adapter_options) do |ldap| + block.call(self.new(provider, ldap)) + end + end + + def self.config(provider) + Gitlab::Auth::LDAP::Config.new(provider) + end + + def initialize(provider, ldap = nil) + @provider = provider + @ldap = ldap || Net::LDAP.new(config.adapter_options) + end + + def config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) + + entries = ldap_search(options).select do |entry| + entry.respond_to? config.uid + end + + entries.map do |entry| + Gitlab::Auth::LDAP::Person.new(entry, provider) + end + end + + def user(*args) + users(*args).first + end + + def dn_matches_filter?(dn, filter) + ldap_search(base: dn, + filter: filter, + scope: Net::LDAP::SearchScope_BaseObject, + attributes: %w{dn}).any? + end + + def ldap_search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) + + if results.nil? + response = ldap.get_operation_result + + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end + + [] + else + results + end + end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] + end + + private + + def user_options(fields, value, limit) + options = { + attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), + base: config.base + } + + options[:size] = limit if limit + + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + else + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) + end + + options.merge(filter: user_filter(filter)) + end + + def user_filter(filter = nil) + user_filter = config.constructed_user_filter if config.user_filter.present? + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb new file mode 100644 index 00000000000..ac5c14d374d --- /dev/null +++ b/lib/gitlab/auth/ldap/auth_hash.rb @@ -0,0 +1,48 @@ +# Class to parse and transform the info provided by omniauth +# +module Gitlab + module Auth + module LDAP + class AuthHash < Gitlab::Auth::OAuth::AuthHash + def uid + @uid ||= Gitlab::Auth::LDAP::Person.normalize_dn(super) + end + + def username + super.tap do |username| + username.downcase! if ldap_config.lowercase_usernames + end + end + + private + + def get_info(key) + attributes = ldap_config.attributes[key.to_s] + return super unless attributes + + attributes = Array(attributes) + + value = nil + attributes.each do |attribute| + value = get_raw(attribute) + value = value.first if value + break if value.present? + end + + return super unless value + + Gitlab::Utils.force_utf8(value) + value + end + + def get_raw(key) + auth_hash.extra[:raw_info][key] if auth_hash.extra + end + + def ldap_config + @ldap_config ||= Gitlab::Auth::LDAP::Config.new(self.provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb new file mode 100644 index 00000000000..e70c3ab6b46 --- /dev/null +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -0,0 +1,68 @@ +# These calls help to authenticate to LDAP by providing username and password +# +# Since multiple LDAP servers are supported, it will loop through all of them +# until a valid bind is found +# + +module Gitlab + module Auth + module LDAP + class Authentication < Gitlab::Auth::OAuth::Authentication + def self.login(login, password) + return unless Gitlab::Auth::LDAP::Config.enabled? + return unless login.present? && password.present? + + auth = nil + # loop through providers until valid bind + providers.find do |provider| + auth = new(provider) + auth.login(login, password) # true will exit the loop + end + + # If (login, password) was invalid for all providers, the value of auth is now the last + # Gitlab::Auth::LDAP::Authentication instance we tried. + auth.user + end + + def self.providers + Gitlab::Auth::LDAP::Config.providers + end + + attr_accessor :ldap_user + + def login(login, password) + @ldap_user = adapter.bind_as( + filter: user_filter(login), + size: 1, + password: password + ) + end + + def adapter + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) + end + + def config + Gitlab::Auth::LDAP::Config.new(provider) + end + + def user_filter(login) + filter = Net::LDAP::Filter.equals(config.uid, login) + + # Apply LDAP user filter if present + if config.user_filter.present? + filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) + end + + filter + end + + def user + return unless ldap_user + + Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb new file mode 100644 index 00000000000..77185f52ced --- /dev/null +++ b/lib/gitlab/auth/ldap/config.rb @@ -0,0 +1,237 @@ +# Load a specific server configuration +module Gitlab + module Auth + module LDAP + class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + + attr_accessor :provider, :options + + def self.enabled? + Gitlab.config.ldap.enabled + end + + def self.servers + Gitlab.config.ldap['servers']&.values || [] + end + + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + + def self.providers + servers.map { |server| server['provider_name'] } + end + + def self.valid_provider?(provider) + providers.include?(provider) + end + + def self.invalid_provider(provider) + raise "Unknown provider (#{provider}). Available providers: #{providers}" + end + + def initialize(provider) + if self.class.valid_provider?(provider) + @provider = provider + else + self.class.invalid_provider(provider) + end + + @options = config_for(@provider) # Use @provider, not provider + end + + def enabled? + base_config.enabled + end + + def adapter_options + opts = base_options.merge( + encryption: encryption_options + ) + + opts.merge!(auth_options) if has_auth? + + opts + end + + def omniauth_options + opts = base_options.merge( + base: base, + encryption: options['encryption'], + filter: omniauth_user_filter, + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] + ) + + if has_auth? + opts.merge!( + bind_dn: options['bind_dn'], + password: options['password'] + ) + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts + end + + def base + options['base'] + end + + def uid + options['uid'] + end + + def sync_ssh_keys? + sync_ssh_keys.present? + end + + # The LDAP attribute in which the ssh keys are stored + def sync_ssh_keys + options['sync_ssh_keys'] + end + + def user_filter + options['user_filter'] + end + + def constructed_user_filter + @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) + end + + def group_base + options['group_base'] + end + + def admin_group + options['admin_group'] + end + + def active_directory + options['active_directory'] + end + + def block_auto_created_users + options['block_auto_created_users'] + end + + def attributes + default_attributes.merge(options['attributes']) + end + + def timeout + options['timeout'].to_i + end + + def has_auth? + options['password'] || options['bind_dn'] + end + + def allow_username_or_email_login + options['allow_username_or_email_login'] + end + + def lowercase_usernames + options['lowercase_usernames'] + end + + def name_proc + if allow_username_or_email_login + proc { |name| name.gsub(/@.*\z/, '') } + else + proc { |name| name } + end + end + + def default_attributes + { + 'username' => %w(uid sAMAccountName userid), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + + protected + + def base_options + { + host: options['host'], + port: options['port'] + } + end + + def base_config + Gitlab.config.ldap + end + + def config_for(provider) + base_config.servers.values.find { |server| server['provider_name'] == provider } + end + + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts + end + + def auth_options + { + auth: { + method: :simple, + username: options['bind_dn'], + password: options['password'] + } + } + end + + def omniauth_user_filter + uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') + + if user_filter.present? + Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s + else + uid_filter.to_s + end + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb new file mode 100644 index 00000000000..1fa5338f5a6 --- /dev/null +++ b/lib/gitlab/auth/ldap/dn.rb @@ -0,0 +1,303 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module Auth + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb new file mode 100644 index 00000000000..8dfae3ee541 --- /dev/null +++ b/lib/gitlab/auth/ldap/person.rb @@ -0,0 +1,122 @@ +module Gitlab + module Auth + module LDAP + class Person + # Active Directory-specific LDAP filter that checks if bit 2 of the + # userAccountControl attribute is set. + # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ + AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + + InvalidEntryError = Class.new(StandardError) + + attr_accessor :entry, :provider + + def self.find_by_uid(uid, adapter) + uid = Net::LDAP::Filter.escape(uid) + adapter.user(adapter.config.uid, uid) + end + + def self.find_by_dn(dn, adapter) + adapter.user('dn', dn) + end + + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + + def self.disabled_via_active_directory?(dn, adapter) + adapter.dn_matches_filter?(dn, AD_USER_DISABLED) + end + + def self.ldap_attributes(config) + [ + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq + end + + def self.normalize_dn(dn) + ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::Auth::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + + def initialize(entry, provider) + Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } + @entry = entry + @provider = provider + end + + def name + attribute_value(:name).first + end + + def uid + entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend + end + + def username + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first.tap do |username| + username.downcase! if config.lowercase_usernames + end + end + + def email + attribute_value(:email) + end + + def dn + self.class.normalize_dn(entry.dn) + end + + private + + def entry + @entry + end + + def config + @config ||= Gitlab::Auth::LDAP::Config.new(provider) + end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_s]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb new file mode 100644 index 00000000000..068212d9a21 --- /dev/null +++ b/lib/gitlab/auth/ldap/user.rb @@ -0,0 +1,54 @@ +# LDAP extension for User model +# +# * Find or create user from omniauth.auth data +# * Links LDAP account with existing user +# * Auth LDAP user with login and password +# +module Gitlab + module Auth + module LDAP + class User < Gitlab::Auth::OAuth::User + class << self + def find_by_uid_and_provider(uid, provider) + identity = ::Identity.with_extern_uid(provider, uid).take + + identity && identity.user + end + end + + def save + super('LDAP') + end + + # instance methods + def find_user + find_by_uid_and_provider || find_by_email || build_new_user + end + + def find_by_uid_and_provider + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) + end + + def changed? + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + + def block_after_signup? + ldap_config.block_auto_created_users + end + + def allowed? + Gitlab::Auth::LDAP::Access.allowed?(gl_user) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(auth_hash.provider) + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Auth::LDAP::AuthHash.new(auth_hash) + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb new file mode 100644 index 00000000000..ed8fba94305 --- /dev/null +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -0,0 +1,92 @@ +# Class to parse and transform the info provided by omniauth +# +module Gitlab + module Auth + module OAuth + class AuthHash + attr_reader :auth_hash + def initialize(auth_hash) + @auth_hash = auth_hash + end + + def uid + @uid ||= Gitlab::Utils.force_utf8(auth_hash.uid.to_s) + end + + def provider + @provider ||= auth_hash.provider.to_s + end + + def name + @name ||= get_info(:name) || "#{get_info(:first_name)} #{get_info(:last_name)}" + end + + def username + @username ||= username_and_email[:username].to_s + end + + def email + @email ||= username_and_email[:email].to_s + end + + def password + @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) + end + + def location + location = get_info(:address) + if location.is_a?(Hash) + [location.locality.presence, location.country.presence].compact.join(', ') + else + location + end + end + + def has_attribute?(attribute) + if attribute == :location + get_info(:address).present? + else + get_info(attribute).present? + end + end + + private + + def info + auth_hash.info + end + + def get_info(key) + value = info[key] + Gitlab::Utils.force_utf8(value) if value + value + end + + def username_and_email + @username_and_email ||= begin + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence + + username ||= generate_username(email) if email + email ||= generate_temporarily_email(username) if username + + { + username: username, + email: email + } + end + end + + # Get the first part of the email address (before @) + # In addtion in removes illegal characters + def generate_username(email) + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s + end + + def generate_temporarily_email(username) + "temp-email-for-oauth-#{username}@gitlab.localhost" + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb new file mode 100644 index 00000000000..ed03b9f8b40 --- /dev/null +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -0,0 +1,21 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module OAuth + class Authentication + attr_reader :provider, :user + + def initialize(provider, user = nil) + @provider = provider + @user = user + end + + def login(login, password) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb new file mode 100644 index 00000000000..5fb61ffe00d --- /dev/null +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -0,0 +1,73 @@ +module Gitlab + module Auth + module OAuth + class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + + def self.authentication(user, provider) + return unless user + return unless enabled?(provider) + + authenticator = + case provider + when /^ldap/ + Gitlab::Auth::LDAP::Authentication + when 'database' + Gitlab::Auth::Database::Authentication + end + + authenticator&.new(provider, user) + end + + def self.providers + Devise.omniauth_providers + end + + def self.enabled?(name) + return true if name == 'database' + + providers.include?(name.to_sym) + end + + def self.ldap_provider?(name) + name.to_s.start_with?('ldap') + end + + def self.sync_profile_from_provider?(provider) + return true if ldap_provider?(provider) + + providers = Gitlab.config.omniauth.sync_profile_from_provider + + if providers.is_a?(Array) + providers.include?(provider) + else + providers + end + end + + def self.config_for(name) + name = name.to_s + if ldap_provider?(name) + if Gitlab::Auth::LDAP::Config.valid_provider?(name) + Gitlab::Auth::LDAP::Config.new(name).options + else + nil + end + else + Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + end + end + + def self.label_for(name) + name = name.to_s + config = config_for(name) + (config && config['label']) || LABELS[name] || name.titleize + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb new file mode 100644 index 00000000000..8f2b4d58552 --- /dev/null +++ b/lib/gitlab/auth/o_auth/session.rb @@ -0,0 +1,21 @@ +# :nocov: +module Gitlab + module Auth + module OAuth + module Session + def self.create(provider, ticket) + Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) + end + + def self.destroy(provider, ticket) + Rails.cache.delete("gitlab:#{provider}:#{ticket}") + end + + def self.valid?(provider, ticket) + Rails.cache.read("gitlab:#{provider}:#{ticket}").present? + end + end + end + end +end +# :nocov: diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb new file mode 100644 index 00000000000..b6a96081278 --- /dev/null +++ b/lib/gitlab/auth/o_auth/user.rb @@ -0,0 +1,246 @@ +# OAuth extension for User model +# +# * Find GitLab user based on omniauth uid and provider +# * Create new user from omniauth data +# +module Gitlab + module Auth + module OAuth + class User + SignupDisabledError = Class.new(StandardError) + SigninDisabledForProviderError = Class.new(StandardError) + + attr_accessor :auth_hash, :gl_user + + def initialize(auth_hash) + self.auth_hash = auth_hash + update_profile + add_or_update_user_identities + end + + def persisted? + gl_user.try(:persisted?) + end + + def new? + !persisted? + end + + def valid? + gl_user.try(:valid?) + end + + def save(provider = 'OAuth') + raise SigninDisabledForProviderError if oauth_provider_disabled? + raise SignupDisabledError unless gl_user + + block_after_save = needs_blocking? + + Users::UpdateService.new(gl_user, user: gl_user).execute! + + gl_user.block if block_after_save + + log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + gl_user + rescue ActiveRecord::RecordInvalid => e + log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" + return self, e.record.errors + end + + def gl_user + return @gl_user if defined?(@gl_user) + + @gl_user = find_user + end + + def find_user + user = find_by_uid_and_provider + + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user&.new_record? + + user + end + + protected + + def add_or_update_user_identities + return unless gl_user + + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user + return unless ldap_person + + user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) + if user + log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." + return user + end + + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) + end + + def auto_link_ldap_user? + Gitlab.config.omniauth.auto_link_ldap_user + end + + def creating_linked_ldap_user? + auto_link_ldap_user? && ldap_person + end + + def ldap_person + return @ldap_person if defined?(@ldap_person) + + # Look for a corresponding person with same uid in any of the configured LDAP providers + Gitlab::Auth::LDAP::Config.providers.each do |provider| + adapter = Gitlab::Auth::LDAP::Adapter.new(provider) + @ldap_person = find_ldap_person(auth_hash, adapter) + break if @ldap_person + end + @ldap_person + end + + def find_ldap_person(auth_hash, adapter) + Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + + def ldap_config + Gitlab::Auth::LDAP::Config.new(ldap_person.provider) if ldap_person + end + + def needs_blocking? + new? && block_after_signup? + end + + def signup_enabled? + providers = Gitlab.config.omniauth.allow_single_sign_on + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end + end + + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + + def block_after_signup? + if creating_linked_ldap_user? + ldap_config.block_auto_created_users + else + Gitlab.config.omniauth.block_auto_created_users + end + end + + def auth_hash=(auth_hash) + @auth_hash = AuthHash.new(auth_hash) + end + + def find_by_uid_and_provider + identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take + identity&.user + end + + def build_new_user + user_params = user_attributes.merge(skip_confirmation: true) + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) + end + + def user_attributes + # Give preference to LDAP for sensitive information when creating a linked account + if creating_linked_ldap_user? + username = ldap_person.username.presence + email = ldap_person.email.first.presence + end + + username ||= auth_hash.username + email ||= auth_hash.email + + valid_username = ::Namespace.clean_path(username) + + uniquify = Uniquify.new + valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } + + name = auth_hash.name + name = valid_username if name.strip.empty? + + { + name: name, + username: valid_username, + email: email, + password: auth_hash.password, + password_confirmation: auth_hash.password, + password_automatically_set: true + } + end + + def sync_profile_from_provider? + Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) + end + + def update_profile + clear_user_synced_attributes_metadata + + return unless sync_profile_from_provider? || creating_linked_ldap_user? + + metadata = gl_user.build_user_synced_attributes_metadata + + if sync_profile_from_provider? + UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) + gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + metadata.set_attribute_synced(key, true) + else + metadata.set_attribute_synced(key, false) + end + end + + metadata.provider = auth_hash.provider + end + + if creating_linked_ldap_user? && gl_user.email == ldap_person.email.first + metadata.set_attribute_synced(:email, true) + metadata.provider = ldap_person.provider + end + end + + def clear_user_synced_attributes_metadata + gl_user&.user_synced_attributes_metadata&.destroy + end + + def log + Gitlab::AppLogger + end + + def oauth_provider_disabled? + Gitlab::CurrentSettings.current_application_settings + .disabled_oauth_sign_in_sources + .include?(auth_hash.provider) + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb new file mode 100644 index 00000000000..c345a7e3f6c --- /dev/null +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Auth + module Saml + class AuthHash < Gitlab::Auth::OAuth::AuthHash + def groups + Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb new file mode 100644 index 00000000000..e654e7fe438 --- /dev/null +++ b/lib/gitlab/auth/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Auth + module Saml + class Config + class << self + def options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb new file mode 100644 index 00000000000..d4024e9ec39 --- /dev/null +++ b/lib/gitlab/auth/saml/user.rb @@ -0,0 +1,52 @@ +# SAML extension for User model +# +# * Find GitLab user based on SAML uid and provider +# * Create new user from SAML data +# +module Gitlab + module Auth + module Saml + class User < Gitlab::Auth::OAuth::User + def save + super('SAML') + end + + def find_user + user = find_by_uid_and_provider + + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + if external_users_enabled? && user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + user.external = !(auth_hash.groups & Gitlab::Auth::Saml::Config.external_groups).empty? + end + + user + end + + def changed? + return true unless gl_user + + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + + protected + + def auto_link_saml_user? + Gitlab.config.omniauth.auto_link_saml_user + end + + def external_users_enabled? + !Gitlab::Auth::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Auth::Saml::AuthHash.new(auth_hash) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb new file mode 100644 index 00000000000..8fe4f1a2289 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class MigrateBuildStage + module Migratable + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + def ensure_stage!(attempts: 2) + find_stage || create_stage! + rescue ActiveRecord::RecordNotUnique + retry if (attempts -= 1) > 0 + raise + end + + def find_stage + Stage.find_by(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + + def create_stage! + Stage.create!(name: self.stage || 'test', + pipeline_id: self.commit_id, + project_id: self.project_id) + end + end + end + + def perform(start_id, stop_id) + stages = Migratable::Build.where('stage_id IS NULL') + .where('id BETWEEN ? AND ?', start_id, stop_id) + .map { |build| build.ensure_stage! } + .compact.map(&:id) + + MigrateBuildStageIdReference.new.perform(start_id, stop_id) + MigrateStageStatus.new.perform(stages.min, stages.max) + end + end + end +end diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index 85749366bfd..d9d3d2e667b 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -16,281 +16,283 @@ module Gitlab # And if the normalize behavior is changed in the future, it must be # accompanied by another migration. module Gitlab - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) + module Auth + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end end - end - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" end - end - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end end - end - str - end + str + end - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end - ## - # Return the DN as an escaped string. - def to_s - @dn - end + ## + # Return the DN as an escaped string. + def to_s + @dn + end - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end - private + private - def initialize_array(args) - buffer = StringIO.new + def initialize_array(args) + buffer = StringIO.new - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end end - end - @dn = buffer.string - end + @dn = buffer.string + end - def initialize_string(arg) - @dn = arg.to_s - end + def initialize_string(arg) + @dn = arg.to_s + end - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end end end end @@ -302,11 +304,11 @@ module Gitlab ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) ldap_identities.each do |identity| begin - identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + identity.extern_uid = Gitlab::Auth::LDAP::DN.new(identity.extern_uid).to_normalized_s unless identity.save Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." end - rescue Gitlab::LDAP::DN::FormatError => e + rescue Gitlab::Auth::LDAP::DN::FormatError => e Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d19a2519803..d5e17a123df 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,27 +17,11 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") - ensure - if pipeline.builds.where(stage_id: nil).any? - invalid_builds_counter.increment(node: hostname) - end end def break? !pipeline.persisted? end - - private - - def invalid_builds_counter - @counter ||= Gitlab::Metrics - .counter(:gitlab_ci_invalid_builds_total, - 'Invalid builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb new file mode 100644 index 00000000000..047ab66e9b3 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Base + def evaluate(**variables) + raise NotImplementedError + end + + def self.build(token) + raise NotImplementedError + end + + def self.scan(scanner) + if scanner.scan(self::PATTERN) + Expression::Token.new(scanner.matched, self) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb new file mode 100644 index 00000000000..3a2f0c6924e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Equals < Lexeme::Operator + PATTERN = /==/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + @left.evaluate(variables) == @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb new file mode 100644 index 00000000000..a2778716924 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Null < Lexeme::Value + PATTERN = /null/.freeze + + def initialize(value = nil) + @value = nil + end + + def evaluate(variables = {}) + nil + end + + def self.build(_value) + self.new + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb new file mode 100644 index 00000000000..f640d0b5855 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Operator < Lexeme::Base + def self.type + :operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb new file mode 100644 index 00000000000..48bde213d44 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class String < Lexeme::Value + PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + + def initialize(value) + @value = value + end + + def evaluate(variables = {}) + @value.to_s + end + + def self.build(string) + new(string.match(PATTERN)[:string]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb new file mode 100644 index 00000000000..f2611d65faf --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Value < Lexeme::Base + def self.type + :value + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb new file mode 100644 index 00000000000..b781c15fd67 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Variable < Lexeme::Value + PATTERN = /\$(?<name>\w+)/.freeze + + def initialize(name) + @name = name + end + + def evaluate(variables = {}) + HashWithIndifferentAccess.new(variables).fetch(@name, nil) + end + + def self.build(string) + new(string.match(PATTERN)[:name]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb new file mode 100644 index 00000000000..e1c68b7c3c2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -0,0 +1,59 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Lexer + include ::Gitlab::Utils::StrongMemoize + + LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Null, + Expression::Lexeme::Equals + ].freeze + + SyntaxError = Class.new(Statement::StatementError) + + MAX_TOKENS = 100 + + def initialize(statement, max_tokens: MAX_TOKENS) + @scanner = StringScanner.new(statement) + @max_tokens = max_tokens + end + + def tokens + strong_memoize(:tokens) { tokenize } + end + + def lexemes + tokens.map(&:to_lexeme) + end + + private + + def tokenize + tokens = [] + + @max_tokens.times do + @scanner.skip(/\s+/) # ignore whitespace + + return tokens if @scanner.eos? + + lexeme = LEXEMES.find do |type| + type.scan(@scanner).tap do |token| + tokens.push(token) if token.present? + end + end + + unless lexeme.present? + raise Lexer::SyntaxError, 'Unknown lexeme found!' + end + end + + raise Lexer::SyntaxError, 'Too many tokens!' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb new file mode 100644 index 00000000000..90f94d0b763 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Parser + def initialize(tokens) + @tokens = tokens.to_enum + @nodes = [] + end + + ## + # This produces a reverse descent parse tree. + # + # It currently does not support precedence of operators. + # + def tree + while token = @tokens.next + case token.type + when :operator + token.build(@nodes.pop, tree).tap do |node| + @nodes.push(node) + end + when :value + token.build.tap do |leaf| + @nodes.push(leaf) + end + end + end + rescue StopIteration + @nodes.last || Lexeme::Null.new + end + + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb new file mode 100644 index 00000000000..4f0e101b730 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -0,0 +1,42 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Statement + StatementError = Class.new(StandardError) + + GRAMMAR = [ + %w[variable equals string], + %w[variable equals variable], + %w[variable equals null], + %w[string equals variable], + %w[null equals variable], + %w[variable] + ].freeze + + def initialize(statement, pipeline) + @lexer = Expression::Lexer.new(statement) + + @variables = pipeline.variables.map do |variable| + [variable.key, variable.value] + end + end + + def parse_tree + raise StatementError if @lexer.lexemes.empty? + + unless GRAMMAR.find { |syntax| syntax == @lexer.lexemes } + raise StatementError, 'Unknown pipeline expression!' + end + + Expression::Parser.new(@lexer.tokens).tree + end + + def evaluate + parse_tree.evaluate(@variables.to_h) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb new file mode 100644 index 00000000000..58211800b88 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -0,0 +1,28 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Token + attr_reader :value, :lexeme + + def initialize(value, lexeme) + @value = value + @lexeme = lexeme + end + + def build(*args) + @lexeme.build(@value, *args) + end + + def type + @lexeme.type + end + + def to_lexeme + @lexeme.name.demodulize.downcase + end + end + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9576d5a3fd8..02d3763514e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -23,7 +23,7 @@ module Gitlab mr_events = event_counts(date_from, :merge_requests) .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") note_events = event_counts(date_from, :merge_requests) - .having(action: [Event::COMMENTED], target_type: "Note") + .having(action: [Event::COMMENTED], target_type: %w(Note DiffNote)) union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 8b3bc3e440d..86d708be0d6 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -8,13 +8,14 @@ module Gitlab private def base_query - @base_query ||= stage_query + @base_query ||= stage_query(@project.id) # rubocop:disable Gitlab/ModuleWithInstanceVariables end - def stage_query + def stage_query(project_ids) query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .project(issue_table[:project_id].as("project_id")) + .where(issue_table[:project_id].in(project_ids)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index cac31ea8cff..038d5a19bc4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -21,17 +21,28 @@ module Gitlab end def median - cte_table = Arel::Table.new("cte_table_for_#{name}") + BatchLoader.for(@project.id).batch(key: name) do |project_ids, loader| + cte_table = Arel::Table.new("cte_table_for_#{name}") - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new( - cte_table, - subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new(cte_table, + subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) - median_datetime(cte_table, interval_query, name) + if project_ids.one? + loader.call(@project.id, median_datetime(cte_table, interval_query, name)) + else + begin + median_datetimes(cte_table, interval_query, name, :project_id)&.each do |project_id, median| + loader.call(project_id, median) + end + rescue NotSupportedError + {} + end + end + end end def name diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index 7a889b3877f..d0ca62e46e4 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -1,8 +1,8 @@ module Gitlab module CycleAnalytics module ProductionHelper - def stage_query - super + def stage_query(project_ids) + super(project_ids) .where(mr_metrics_table[:first_deployed_to_production_at] .gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index 2b5f72bef89..0e9d235ca79 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -25,11 +25,11 @@ module Gitlab _("Total test time for all commits/merges") end - def stage_query + def stage_query(project_ids) if @options[:branch] - super.where(build_table[:ref].eq(@options[:branch])) + super(project_ids).where(build_table[:ref].eq(@options[:branch])) else - super + super(project_ids) end end end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb new file mode 100644 index 00000000000..5122e3417ca --- /dev/null +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -0,0 +1,72 @@ +module Gitlab + module CycleAnalytics + class UsageData + PROJECTS_LIMIT = 10 + + attr_reader :projects, :options + + def initialize + @projects = Project.sorted_by_activity.limit(PROJECTS_LIMIT) + @options = { from: 7.days.ago } + end + + def to_json + total = 0 + + values = + medians_per_stage.each_with_object({}) do |(stage_name, medians), hsh| + calculations = stage_values(medians) + + total += calculations.values.compact.sum + hsh[stage_name] = calculations + end + + values[:total] = total + + { avg_cycle_analytics: values } + end + + private + + def medians_per_stage + projects.each_with_object({}) do |project, hsh| + ::CycleAnalytics.new(project, options).all_medians_per_stage.each do |stage_name, median| + hsh[stage_name] ||= [] + hsh[stage_name] << median + end + end + end + + def stage_values(medians) + medians = medians.map(&:presence).compact + average = calc_average(medians) + + { + average: average, + sd: standard_deviation(medians, average), + missing: projects.length - medians.length + } + end + + def calc_average(values) + return if values.empty? + + (values.sum / values.length).to_i + end + + def standard_deviation(values, average) + Math.sqrt(sample_variance(values, average)).to_i + end + + def sample_variance(values, average) + return 0 if values.length <= 1 + + sum = values.inject(0) do |acc, val| + acc + (val - average)**2 + end + + sum / (values.length - 1) + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 059054ac9ff..74fed447289 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -2,18 +2,14 @@ module Gitlab module Database module Median + NotSupportedError = Class.new(StandardError) + def median_datetime(arel_table, query_so_far, column_sym) - median_queries = - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym) - elsif Gitlab::Database.mysql? - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - - results = Array.wrap(median_queries).map do |query| - ActiveRecord::Base.connection.execute(query) - end - extract_median(results).presence + extract_median(execute_queries(arel_table, query_so_far, column_sym)).presence + end + + def median_datetimes(arel_table, query_so_far, column_sym, partition_column) + extract_medians(execute_queries(arel_table, query_so_far, column_sym, partition_column)).presence end def extract_median(results) @@ -21,13 +17,21 @@ module Gitlab if Gitlab::Database.postgresql? result = result.first.presence - median = result['median'] if result - median.to_f if median + + result['median']&.to_f if result elsif Gitlab::Database.mysql? result.to_a.flatten.first end end + def extract_medians(results) + median_values = results.compact.first.values + + median_values.each_with_object({}) do |(id, median), hash| + hash[id.to_i] = median&.to_f + end + end + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) query = arel_table .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) @@ -53,7 +57,7 @@ module Gitlab ] end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows # of the CTE. For example, if we're looking to find the median of the `projects.star_count` @@ -64,41 +68,107 @@ module Gitlab # 5 | 1 | 3 # 9 | 2 | 3 # 15 | 3 | 3 + # + # If a partition column is used we will do the same operation but for separate partitions, + # when that happens the CTE might look like this: + # + # project_id | star_count | row_id | ct + # ------------+------------+--------+---- + # 1 | 5 | 1 | 2 + # 1 | 9 | 2 | 2 + # 2 | 10 | 1 | 3 + # 2 | 15 | 2 | 3 + # 2 | 20 | 3 | 3 cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( cte_table, - arel_table - .project( - arel_table[column_sym].as(column_sym.to_s), - Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), - Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), - arel_table.project("COUNT(1)").as('ct')). + arel_table.project(*rank_rows(arel_table, column_sym, partition_column)). # Disallow negative values where(arel_table[column_sym].gteq(zero_interval))) # From the CTE, select either the middle row or the middle two rows (this is accomplished # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the # selected rows, and this is the median value. - cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")) - .where( - Arel::Nodes::Between.new( - cte_table[:row_id], - Arel::Nodes::And.new( - [(cte_table[:ct] / Arel.sql('2.0')), - (cte_table[:ct] / Arel.sql('2.0') + 1)] + result = + cte_table + .project(*median_projections(cte_table, column_sym, partition_column)) + .where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) ) ) - ) - .with(query_so_far, cte) - .to_sql + .with(query_so_far, cte) + + result.group(cte_table[partition_column]).order(cte_table[partition_column]) if partition_column + + result.to_sql end private + def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) + elsif Gitlab::Database.mysql? + raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column + + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + end + + def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) + queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + + Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } + end + def average(args, as) Arel::Nodes::NamedFunction.new("AVG", args, as) end + def rank_rows(arel_table, column_sym, partition_column) + column_row = arel_table[column_sym].as(column_sym.to_s) + + if partition_column + partition_row = arel_table[partition_column] + row_id = + Arel::Nodes::Over.new( + Arel::Nodes::NamedFunction.new('rank', []), + Arel::Nodes::Window.new.partition(arel_table[partition_column]) + .order(arel_table[column_sym]) + ).as('row_id') + + count = arel_table.from(arel_table.alias) + .project('COUNT(*)') + .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) + .as('ct') + + [partition_row, column_row, row_id, count] + else + row_id = + Arel::Nodes::Over.new( + Arel::Nodes::NamedFunction.new('row_number', []), + Arel::Nodes::Window.new.order(arel_table[column_sym]) + ).as('row_id') + + count = arel_table.project("COUNT(1)").as('ct') + + [column_row, row_id, count] + end + end + + def median_projections(table, column_sym, partition_column) + projections = [] + projections << table[partition_column] if partition_column + projections << average([extract_epoch(table[column_sym])], "median") + projections + end + def extract_epoch(arel_attribute) Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b2fca2c16de..eabcf46cf58 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -238,9 +238,9 @@ module Gitlab self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end - @loaded_all_data = false # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data + @loaded_all_data = @loaded_size == size end def binary? @@ -255,10 +255,15 @@ module Gitlab # memory as a Ruby string. def load_all_data!(repository) return if @data == '' # don't mess with submodule blobs - return @data if @loaded_all_data - Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - @data = begin + # Even if we return early, recalculate wether this blob is binary in + # case a blob was initialized as text but the full data isn't + @binary = nil + + return if @loaded_all_data + + @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + begin if is_enabled repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else @@ -269,7 +274,6 @@ module Gitlab @loaded_all_data = true @loaded_size = @data.bytesize - @binary = nil end def name diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index ae7e88f0503..6351cfb83e3 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Branch < Ref + STALE_BRANCH_THRESHOLD = 3.months + def self.find(repo, branch_name) if branch_name.is_a?(Gitlab::Git::Branch) branch_name @@ -12,6 +14,18 @@ module Gitlab def initialize(repository, name, target, target_commit) super(repository, name, target, target_commit) end + + def active? + self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago + end + + def stale? + !active? + end + + def state + active? ? :active : :stale + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index ae27a138b7c..594b6a9cbc5 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -250,6 +250,45 @@ module Gitlab end end + def extract_signature_lazily(repository, commit_id) + BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + commit_ids = items.map { |i| i[:commit_id] } + + signatures = batch_signature_extraction(repository, commit_ids) + + signatures.each do |commit_sha, signature_data| + loader.call({ repository: repository, commit_id: commit_sha }, signature_data) + end + end + end + end + + def batch_signature_extraction(repository, commit_ids) + repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| + if is_enabled + gitaly_batch_signature_extraction(repository, commit_ids) + else + rugged_batch_signature_extraction(repository, commit_ids) + end + end + end + + def gitaly_batch_signature_extraction(repository, commit_ids) + repository.gitaly_commit_client.get_commit_signatures(commit_ids) + end + + def rugged_batch_signature_extraction(repository, commit_ids) + commit_ids.each_with_object({}) do |commit_id, signatures| + signature_data = rugged_extract_signature(repository, commit_id) + next unless signature_data + + signatures[commit_id] = signature_data + end + end + def rugged_extract_signature(repository, commit_id) begin Rugged::Commit.extract_signature(repository.rugged, commit_id) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 48434047fce..b9e5cf258f4 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,6 +7,28 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) + @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + else + git_new_pointers(object_limit, not_in) + end + end + end + + def all_pointers + @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) + else + git_all_pointers + end + end + end + + private + + def git_new_pointers(object_limit, not_in) @new_pointers ||= begin rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| object_ids = object_ids.take(object_limit) if object_limit @@ -16,14 +38,12 @@ module Gitlab end end - def all_pointers + def git_all_pointers rev_list.all_objects(require_path: true) do |object_ids| Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) end end - private - def rev_list Gitlab::Git::RevList.new(@repository, newrev: @newrev) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e3cbf017e55..21c79a7a550 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -467,7 +467,8 @@ module Gitlab follow: false, skip_merges: false, after: nil, - before: nil + before: nil, + all: false } options = default_options.merge(options) @@ -489,13 +490,16 @@ module Gitlab # Used in gitaly-ruby def raw_log(options) - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end + sha = + unless options[:all] + actual_ref = options[:ref] || root_ref + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + end log_by_shell(sha, options) end @@ -1032,6 +1036,21 @@ module Gitlab end end + def license_short_name + gitaly_migrate(:license_short_name) do |is_enabled| + if is_enabled + gitaly_repository_client.license_short_name + else + begin + # The licensee gem creates a Rugged object from the path: + # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb + Licensee.license(path).try(:key) + rescue Rugged::Error + end + end + end + end + def with_repo_branch_commit(start_repository, start_branch_name) Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) @@ -1701,7 +1720,12 @@ module Gitlab cmd << '--no-merges' if options[:skip_merges] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha + + if options[:all] + cmd += %w[--all --reverse] + else + cmd << sha + end # :path can be a string or an array of strings if options[:path].present? @@ -1918,7 +1942,16 @@ module Gitlab cmd << "--before=#{options[:before].iso8601}" if options[:before] cmd << "--max-count=#{options[:max_count]}" if options[:max_count] cmd << "--left-right" if options[:left_right] - cmd += %W[--count #{options[:ref]}] + cmd << '--count' + + cmd << if options[:all] + '--all' + elsif options[:ref] + options[:ref] + else + raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" + end + cmd += %W[-- #{options[:path]}] if options[:path].present? cmd end @@ -2206,7 +2239,7 @@ module Gitlab with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do # Apply diff of the `diff_range` to the worktree diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| stdin.binmode stdin.write(diff) end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index ba6058fd3c9..b6ceb542dd1 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -14,14 +14,14 @@ module Gitlab # Uses rugged for raw objects # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 - def where(repository, sha, path = nil) + def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - repository.gitaly_commit_client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) else - tree_entries_from_rugged(repository, sha, path) + tree_entries_from_rugged(repository, sha, path, recursive) end end end @@ -57,7 +57,22 @@ module Gitlab end end - def tree_entries_from_rugged(repository, sha, path) + def tree_entries_from_rugged(repository, sha, path, recursive) + current_path_entries = get_tree_entries_from_rugged(repository, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) + end + end + + ordered_entries + end + + def get_tree_entries_from_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index ac12271a87e..52b44b9b3c5 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -59,7 +59,7 @@ module Gitlab end def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| if is_enabled gitaly_get_all_pages else @@ -68,9 +68,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039 def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| if is_enabled gitaly_find_page(title: title, version: version, dir: dir) else diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bbdb593d4e2..6400089a22f 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -199,7 +199,7 @@ module Gitlab def check_repository_existence! unless repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + raise NotFoundError, ERROR_MESSAGES[:no_repo] end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c5d3e944f7d..9cd76630484 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -125,6 +125,8 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + rescue GRPC::Unavailable => ex + handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -135,6 +137,27 @@ module Gitlab duration) end + def self.handle_grpc_unavailable!(ex) + status = ex.to_status + raise ex unless status.details == 'Endpoint read failed' + + # There is a bug in grpc 1.8.x that causes a client process to get stuck + # always raising '14:Endpoint read failed'. The only thing that we can + # do to recover is to restart the process. + # + # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + + if Sidekiq.server? + raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) + else + # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. + Process.kill('QUIT', Process.pid) + end + + raise ex + end + private_class_method :handle_grpc_unavailable! + def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index dfa0fa43b0f..28554208984 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -45,16 +45,7 @@ module Gitlab response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) - response.flat_map do |message| - message.lfs_pointers.map do |lfs_pointer| - Gitlab::Git::Blob.new( - id: lfs_pointer.oid, - size: lfs_pointer.size, - data: lfs_pointer.data, - binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) - ) - end - end + map_lfs_pointers(response) end def get_blobs(revision_paths, limit = -1) @@ -80,6 +71,50 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end + + def get_new_lfs_pointers(revision, limit, not_in) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + limit: limit || 0 + ) + + if not_in.nil? || not_in == :all + request.not_in_all = true + else + request.not_in_refs += not_in + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + + map_lfs_pointers(response) + end + + def get_all_lfs_pointers(revision) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + + map_lfs_pointers(response) + end + + private + + def map_lfs_pointers(response) + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 269a048cf5d..456a8a1a2d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -105,11 +105,12 @@ module Gitlab entry unless entry.oid.blank? end - def tree_entries(repository, revision, path) + def tree_entries(repository, revision, path, recursive) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), - path: path.present? ? encode_binary(path) : '.' + path: path.present? ? encode_binary(path) : '.', + recursive: recursive ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -133,7 +134,8 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: encode_binary(ref) + revision: encode_binary(ref), + all: !!options[:all] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -268,6 +270,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], + all: !!options[:all], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] @@ -318,6 +321,23 @@ module Gitlab [signature, signed_text] end + def get_commit_signatures(commit_ids) + request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request) + + signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] } + current_commit_id = nil + + response.each do |message| + current_commit_id = message.commit_id if message.commit_id.present? + + signatures[current_commit_id].first << message.signature + signatures[current_commit_id].last << message.signed_text + end + + signatures + end + private def call_commit_diff(request_params, options = {}) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 603457d0664..fdb3247cf4d 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -41,7 +41,7 @@ module Gitlab end def apply_gitattributes(revision) - request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) + request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end @@ -249,6 +249,14 @@ module Gitlab raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? end + + def license_short_name + request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) + + response = GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.fast_timeout) + + response.license_short_name.presence + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ba04387022d..a7e055ac444 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -19,6 +19,8 @@ module Gitlab gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path + gon.test_env = Rails.env.test? + gon.suggested_label_colors = LabelsHelper.suggested_colors if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 90dd569aaf8..6d2278d0876 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,15 +1,29 @@ module Gitlab module Gpg class Commit + include Gitlab::Utils::StrongMemoize + def initialize(commit) @commit = commit repo = commit.project.repository.raw_repository - @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha) + @signature_data = Gitlab::Git::Commit.extract_signature_lazily(repo, commit.sha || commit.id) + end + + def signature_text + strong_memoize(:signature_text) do + @signature_data&.itself && @signature_data[0] + end + end + + def signed_text + strong_memoize(:signed_text) do + @signature_data&.itself && @signature_data[1] + end end def has_signature? - !!(@signature_text && @signed_text) + !!(signature_text && signed_text) end def signature @@ -53,7 +67,7 @@ module Gitlab end def verified_signature - @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| break verified_signature end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index af203ff711d..b713fa7e1cd 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.2'.freeze + VERSION = '0.2.3'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9f404003125..4bdd01f5e94 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -65,6 +65,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :project_badges # Only include the following attributes for the models specified. included_attributes: @@ -125,6 +126,8 @@ excluded_attributes: - :when push_event_payload: - :event_id + project_badges: + - :group_id methods: labels: @@ -147,3 +150,5 @@ methods: - :action push_event_payload: - :action + project_badges: + - :type diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 759833a5ee5..cf6b7e306dd 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -16,7 +16,8 @@ module Gitlab priorities: :label_priorities, auto_devops: :project_auto_devops, label: :project_label, - custom_attributes: 'ProjectCustomAttribute' }.freeze + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index f654508c391..f7a8eae0be4 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -15,16 +15,22 @@ module Gitlab # push to that array when done. Once the waiter has popped `count` items, it # knows all the jobs are done. class JobWaiter + KEY_PREFIX = "gitlab:job_waiter".freeze + def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end + def self.key?(key) + key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ + end + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] diff --git a/lib/gitlab/kubernetes/config_map.rb b/lib/gitlab/kubernetes/config_map.rb new file mode 100644 index 00000000000..95e1054919d --- /dev/null +++ b/lib/gitlab/kubernetes/config_map.rb @@ -0,0 +1,37 @@ +module Gitlab + module Kubernetes + class ConfigMap + def initialize(name, values) + @name = name + @values = values + end + + def generate + resource = ::Kubeclient::Resource.new + resource.metadata = metadata + resource.data = { values: values } + resource + end + + private + + attr_reader :name, :values + + def metadata + { + name: config_map_name, + namespace: namespace, + labels: { name: config_map_name } + } + end + + def config_map_name + "values-content-configuration-#{name}" + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 737081ddc5b..2edd34109ba 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -9,7 +9,8 @@ module Gitlab def install(command) @namespace.ensure_exists! - @kubeclient.create_pod(pod_resource(command)) + create_config_map(command) if command.config_map? + @kubeclient.create_pod(command.pod_resource) end ## @@ -33,8 +34,10 @@ module Gitlab private - def pod_resource(command) - Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate + def create_config_map(command) + command.config_map_resource.tap do |config_map_resource| + @kubeclient.create_config_map(config_map_resource) + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb new file mode 100644 index 00000000000..6e4df05aa7e --- /dev/null +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -0,0 +1,40 @@ +module Gitlab + module Kubernetes + module Helm + class BaseCommand + attr_reader :name + + def initialize(name) + @name = name + end + + def pod_resource + Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate + end + + def generate_script + <<~HEREDOC + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + HEREDOC + end + + def config_map? + false + end + + def pod_name + "install-#{name}" + end + + private + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb new file mode 100644 index 00000000000..a02e64561f6 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -0,0 +1,19 @@ +module Gitlab + module Kubernetes + module Helm + class InitCommand < BaseCommand + def generate_script + super + [ + init_helm_command + ].join("\n") + end + + private + + def init_helm_command + "helm init >/dev/null" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index bf6981035f4..30af3e97b4a 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -1,54 +1,45 @@ module Gitlab module Kubernetes module Helm - class InstallCommand - attr_reader :name, :install_helm, :chart, :chart_values_file + class InstallCommand < BaseCommand + attr_reader :name, :chart, :repository, :values - def initialize(name, install_helm: false, chart: false, chart_values_file: false) + def initialize(name, chart:, values:, repository: nil) @name = name - @install_helm = install_helm @chart = chart - @chart_values_file = chart_values_file + @values = values + @repository = repository end - def pod_name - "install-#{name}" + def generate_script + super + [ + init_command, + repository_command, + script_command + ].compact.join("\n") end - def generate_script(namespace_name) - [ - install_dps_command, - init_command, - complete_command(namespace_name) - ].join("\n") + def config_map? + true + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, values).generate end private def init_command - if install_helm - 'helm init >/dev/null' - else - 'helm init --client-only >/dev/null' - end + 'helm init --client-only >/dev/null' end - def complete_command(namespace_name) - return unless chart - - if chart_values_file - "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null" - else - "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" - end + def repository_command + "helm repo add #{name} #{repository}" if repository end - def install_dps_command + def script_command <<~HEREDOC - set -eo pipefail - apk add -U ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ + helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null HEREDOC end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index ca5e06009fa..1e12299eefd 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -2,18 +2,17 @@ module Gitlab module Kubernetes module Helm class Pod - def initialize(command, namespace_name, kubeclient) + def initialize(command, namespace_name) @command = command @namespace_name = namespace_name - @kubeclient = kubeclient end def generate spec = { containers: [container_specification], restartPolicy: 'Never' } - if command.chart_values_file - create_config_map + if command.config_map? spec[:volumes] = volumes_specification + spec[:containers][0][:volumeMounts] = volume_mounts_specification end ::Kubeclient::Resource.new(metadata: metadata, spec: spec) @@ -21,18 +20,16 @@ module Gitlab private - attr_reader :command, :namespace_name, :kubeclient + attr_reader :command, :namespace_name, :kubeclient, :config_map def container_specification - container = { + { name: 'helm', image: 'alpine:3.6', env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) } - container[:volumeMounts] = volume_mounts_specification if command.chart_values_file - container end def labels @@ -50,13 +47,12 @@ module Gitlab } end - def volume_mounts_specification - [ - { - name: 'configuration-volume', - mountPath: "/data/helm/#{command.name}/config" - } - ] + def generate_pod_env(command) + { + HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, + TILLER_NAMESPACE: namespace_name, + COMMAND_SCRIPT: command.generate_script + }.map { |key, value| { name: key, value: value } } end def volumes_specification @@ -71,23 +67,13 @@ module Gitlab ] end - def generate_pod_env(command) - { - HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, - TILLER_NAMESPACE: namespace_name, - COMMAND_SCRIPT: command.generate_script(namespace_name) - }.map { |key, value| { name: key, value: value } } - end - - def create_config_map - resource = ::Kubeclient::Resource.new - resource.metadata = { - name: "values-content-configuration-#{command.name}", - namespace: namespace_name, - labels: { name: "values-content-configuration-#{command.name}" } - } - resource.data = { values: File.read(command.chart_values_file) } - kubeclient.create_config_map(resource) + def volume_mounts_specification + [ + { + name: 'configuration-volume', + mountPath: "/data/helm/#{command.name}/config" + } + ] end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb deleted file mode 100644 index e60ceba27c8..00000000000 --- a/lib/gitlab/ldap/access.rb +++ /dev/null @@ -1,87 +0,0 @@ -# LDAP authorization model -# -# * Check if we are allowed access (not blocked) -# -module Gitlab - module LDAP - class Access - attr_reader :provider, :user - - def self.open(user, &block) - Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| - block.call(self.new(user, adapter)) - end - end - - def self.allowed?(user) - self.open(user) do |access| - if access.allowed? - Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute - - true - else - false - end - end - end - - def initialize(user, adapter = nil) - @adapter = adapter - @user = user - @provider = user.ldap_identity.provider - end - - def allowed? - if ldap_user - unless ldap_config.active_directory - unblock_user(user, 'is available again') if user.ldap_blocked? - return true - end - - # Block user in GitLab if he/she was blocked in AD - if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - block_user(user, 'is disabled in Active Directory') - false - else - unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? - true - end - else - # Block the user if they no longer exist in LDAP/AD - block_user(user, 'does not exist anymore') - false - end - end - - def adapter - @adapter ||= Gitlab::LDAP::Adapter.new(provider) - end - - def ldap_config - Gitlab::LDAP::Config.new(provider) - end - - def ldap_user - @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) - end - - def block_user(user, reason) - user.ldap_block - - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" - ) - end - - def unblock_user(user, reason) - user.activate - - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" - ) - end - end - end -end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb deleted file mode 100644 index 76863e77dc3..00000000000 --- a/lib/gitlab/ldap/adapter.rb +++ /dev/null @@ -1,108 +0,0 @@ -module Gitlab - module LDAP - class Adapter - attr_reader :provider, :ldap - - def self.open(provider, &block) - Net::LDAP.open(config(provider).adapter_options) do |ldap| - block.call(self.new(provider, ldap)) - end - end - - def self.config(provider) - Gitlab::LDAP::Config.new(provider) - end - - def initialize(provider, ldap = nil) - @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) - end - - def config - Gitlab::LDAP::Config.new(provider) - end - - def users(fields, value, limit = nil) - options = user_options(Array(fields), value, limit) - - entries = ldap_search(options).select do |entry| - entry.respond_to? config.uid - end - - entries.map do |entry| - Gitlab::LDAP::Person.new(entry, provider) - end - end - - def user(*args) - users(*args).first - end - - def dn_matches_filter?(dn, filter) - ldap_search(base: dn, - filter: filter, - scope: Net::LDAP::SearchScope_BaseObject, - attributes: %w{dn}).any? - end - - def ldap_search(*args) - # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do - results = ldap.search(*args) - - if results.nil? - response = ldap.get_operation_result - - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end - - [] - else - results - end - end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] - end - - private - - def user_options(fields, value, limit) - options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config), - base: config.base - } - - options[:size] = limit if limit - - if fields.include?('dn') - raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 - - options[:base] = value - options[:scope] = Net::LDAP::SearchScope_BaseObject - else - filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) - end - - options.merge(filter: user_filter(filter)) - end - - def user_filter(filter = nil) - user_filter = config.constructed_user_filter if config.user_filter.present? - - if user_filter && filter - Net::LDAP::Filter.join(filter, user_filter) - elsif user_filter - user_filter - else - filter - end - end - end - end -end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb deleted file mode 100644 index 96171dc26c4..00000000000 --- a/lib/gitlab/ldap/auth_hash.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Class to parse and transform the info provided by omniauth -# -module Gitlab - module LDAP - class AuthHash < Gitlab::OAuth::AuthHash - def uid - @uid ||= Gitlab::LDAP::Person.normalize_dn(super) - end - - def username - super.tap do |username| - username.downcase! if ldap_config.lowercase_usernames - end - end - - private - - def get_info(key) - attributes = ldap_config.attributes[key.to_s] - return super unless attributes - - attributes = Array(attributes) - - value = nil - attributes.each do |attribute| - value = get_raw(attribute) - value = value.first if value - break if value.present? - end - - return super unless value - - Gitlab::Utils.force_utf8(value) - value - end - - def get_raw(key) - auth_hash.extra[:raw_info][key] if auth_hash.extra - end - - def ldap_config - @ldap_config ||= Gitlab::LDAP::Config.new(self.provider) - end - end - end -end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb deleted file mode 100644 index 7274d1c3b43..00000000000 --- a/lib/gitlab/ldap/authentication.rb +++ /dev/null @@ -1,70 +0,0 @@ -# These calls help to authenticate to LDAP by providing username and password -# -# Since multiple LDAP servers are supported, it will loop through all of them -# until a valid bind is found -# - -module Gitlab - module LDAP - class Authentication - def self.login(login, password) - return unless Gitlab::LDAP::Config.enabled? - return unless login.present? && password.present? - - auth = nil - # loop through providers until valid bind - providers.find do |provider| - auth = new(provider) - auth.login(login, password) # true will exit the loop - end - - # If (login, password) was invalid for all providers, the value of auth is now the last - # Gitlab::LDAP::Authentication instance we tried. - auth.user - end - - def self.providers - Gitlab::LDAP::Config.providers - end - - attr_accessor :provider, :ldap_user - - def initialize(provider) - @provider = provider - end - - def login(login, password) - @ldap_user = adapter.bind_as( - filter: user_filter(login), - size: 1, - password: password - ) - end - - def adapter - OmniAuth::LDAP::Adaptor.new(config.omniauth_options) - end - - def config - Gitlab::LDAP::Config.new(provider) - end - - def user_filter(login) - filter = Net::LDAP::Filter.equals(config.uid, login) - - # Apply LDAP user filter if present - if config.user_filter.present? - filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) - end - - filter - end - - def user - return nil unless ldap_user - - Gitlab::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) - end - end - end -end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb deleted file mode 100644 index a6bea98d631..00000000000 --- a/lib/gitlab/ldap/config.rb +++ /dev/null @@ -1,235 +0,0 @@ -# Load a specific server configuration -module Gitlab - module LDAP - class Config - NET_LDAP_ENCRYPTION_METHOD = { - simple_tls: :simple_tls, - start_tls: :start_tls, - plain: nil - }.freeze - - attr_accessor :provider, :options - - def self.enabled? - Gitlab.config.ldap.enabled - end - - def self.servers - Gitlab.config.ldap['servers']&.values || [] - end - - def self.available_servers - return [] unless enabled? - - Array.wrap(servers.first) - end - - def self.providers - servers.map { |server| server['provider_name'] } - end - - def self.valid_provider?(provider) - providers.include?(provider) - end - - def self.invalid_provider(provider) - raise "Unknown provider (#{provider}). Available providers: #{providers}" - end - - def initialize(provider) - if self.class.valid_provider?(provider) - @provider = provider - else - self.class.invalid_provider(provider) - end - - @options = config_for(@provider) # Use @provider, not provider - end - - def enabled? - base_config.enabled - end - - def adapter_options - opts = base_options.merge( - encryption: encryption_options - ) - - opts.merge!(auth_options) if has_auth? - - opts - end - - def omniauth_options - opts = base_options.merge( - base: base, - encryption: options['encryption'], - filter: omniauth_user_filter, - name_proc: name_proc, - disable_verify_certificates: !options['verify_certificates'] - ) - - if has_auth? - opts.merge!( - bind_dn: options['bind_dn'], - password: options['password'] - ) - end - - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - - opts - end - - def base - options['base'] - end - - def uid - options['uid'] - end - - def sync_ssh_keys? - sync_ssh_keys.present? - end - - # The LDAP attribute in which the ssh keys are stored - def sync_ssh_keys - options['sync_ssh_keys'] - end - - def user_filter - options['user_filter'] - end - - def constructed_user_filter - @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) - end - - def group_base - options['group_base'] - end - - def admin_group - options['admin_group'] - end - - def active_directory - options['active_directory'] - end - - def block_auto_created_users - options['block_auto_created_users'] - end - - def attributes - default_attributes.merge(options['attributes']) - end - - def timeout - options['timeout'].to_i - end - - def has_auth? - options['password'] || options['bind_dn'] - end - - def allow_username_or_email_login - options['allow_username_or_email_login'] - end - - def lowercase_usernames - options['lowercase_usernames'] - end - - def name_proc - if allow_username_or_email_login - proc { |name| name.gsub(/@.*\z/, '') } - else - proc { |name| name } - end - end - - def default_attributes - { - 'username' => %w(uid sAMAccountName userid), - 'email' => %w(mail email userPrincipalName), - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' - } - end - - protected - - def base_options - { - host: options['host'], - port: options['port'] - } - end - - def base_config - Gitlab.config.ldap - end - - def config_for(provider) - base_config.servers.values.find { |server| server['provider_name'] == provider } - end - - def encryption_options - method = translate_method(options['encryption']) - return nil unless method - - { - method: method, - tls_options: tls_options(method) - } - end - - def translate_method(method_from_config) - NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] - end - - def tls_options(method) - return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method - - opts = if options['verify_certificates'] - OpenSSL::SSL::SSLContext::DEFAULT_PARAMS - else - # It is important to explicitly set verify_mode for two reasons: - # 1. The behavior of OpenSSL is undefined when verify_mode is not set. - # 2. The net-ldap gem implementation verifies the certificate hostname - # unless verify_mode is set to VERIFY_NONE. - { verify_mode: OpenSSL::SSL::VERIFY_NONE } - end - - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - - opts - end - - def auth_options - { - auth: { - method: :simple, - username: options['bind_dn'], - password: options['password'] - } - } - end - - def omniauth_user_filter - uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') - - if user_filter.present? - Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s - else - uid_filter.to_s - end - end - end - end -end diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb deleted file mode 100644 index d6142dc6549..00000000000 --- a/lib/gitlab/ldap/dn.rb +++ /dev/null @@ -1,301 +0,0 @@ -# -*- ruby encoding: utf-8 -*- - -# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` -# -# For our purposes, this class is used to normalize DNs in order to allow proper -# comparison. -# -# E.g. DNs should be compared case-insensitively (in basically all LDAP -# implementations or setups), therefore we downcase every DN. - -## -# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN -# ("Distinguished Name") is a unique identifier for an entry within an LDAP -# directory. It is made up of a number of other attributes strung together, -# to identify the entry in the tree. -# -# Each attribute that makes up a DN needs to have its value escaped so that -# the DN is valid. This class helps take care of that. -# -# A fully escaped DN needs to be unescaped when analysing its contents. This -# class also helps take care of that. -module Gitlab - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) - - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end - - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) - end - end - - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" - - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char - end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" - end - end - - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state - - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end - - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) - - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! - end - end - - str - end - - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end - - ## - # Return the DN as an escaped string. - def to_s - @dn - end - - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end - - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze - - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") - - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") - - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end - - private - - def initialize_array(args) - buffer = StringIO.new - - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) - end - end - - @dn = buffer.string - end - - def initialize_string(arg) - @dn = arg.to_s - end - - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end - - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) - end - end - end -end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb deleted file mode 100644 index c59df556247..00000000000 --- a/lib/gitlab/ldap/person.rb +++ /dev/null @@ -1,120 +0,0 @@ -module Gitlab - module LDAP - class Person - # Active Directory-specific LDAP filter that checks if bit 2 of the - # userAccountControl attribute is set. - # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ - AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") - - InvalidEntryError = Class.new(StandardError) - - attr_accessor :entry, :provider - - def self.find_by_uid(uid, adapter) - uid = Net::LDAP::Filter.escape(uid) - adapter.user(adapter.config.uid, uid) - end - - def self.find_by_dn(dn, adapter) - adapter.user('dn', dn) - end - - def self.find_by_email(email, adapter) - email_fields = adapter.config.attributes['email'] - - adapter.user(email_fields, email) - end - - def self.disabled_via_active_directory?(dn, adapter) - adapter.dn_matches_filter?(dn, AD_USER_DISABLED) - end - - def self.ldap_attributes(config) - [ - 'dn', - config.uid, - *config.attributes['name'], - *config.attributes['email'], - *config.attributes['username'] - ].compact.uniq - end - - def self.normalize_dn(dn) - ::Gitlab::LDAP::DN.new(dn).to_normalized_s - rescue ::Gitlab::LDAP::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") - - dn - end - - # Returns the UID in a normalized form. - # - # 1. Excess spaces are stripped - # 2. The string is downcased (for case-insensitivity) - def self.normalize_uid(uid) - ::Gitlab::LDAP::DN.normalize_value(uid) - rescue ::Gitlab::LDAP::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") - - uid - end - - def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } - @entry = entry - @provider = provider - end - - def name - attribute_value(:name).first - end - - def uid - entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend - end - - def username - username = attribute_value(:username) - - # Depending on the attribute, multiple values may - # be returned. We need only one for username. - # Ex. `uid` returns only one value but `mail` may - # return an array of multiple email addresses. - [username].flatten.first.tap do |username| - username.downcase! if config.lowercase_usernames - end - end - - def email - attribute_value(:email) - end - - def dn - self.class.normalize_dn(entry.dn) - end - - private - - def entry - @entry - end - - def config - @config ||= Gitlab::LDAP::Config.new(provider) - end - - # Using the LDAP attributes configuration, find and return the first - # attribute with a value. For example, by default, when given 'email', - # this method looks for 'mail', 'email' and 'userPrincipalName' and - # returns the first with a value. - def attribute_value(attribute) - attributes = Array(config.attributes[attribute.to_s]) - selected_attr = attributes.find { |attr| entry.respond_to?(attr) } - - return nil unless selected_attr - - entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend - end - end - end -end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb deleted file mode 100644 index 84ee94e38e4..00000000000 --- a/lib/gitlab/ldap/user.rb +++ /dev/null @@ -1,52 +0,0 @@ -# LDAP extension for User model -# -# * Find or create user from omniauth.auth data -# * Links LDAP account with existing user -# * Auth LDAP user with login and password -# -module Gitlab - module LDAP - class User < Gitlab::OAuth::User - class << self - def find_by_uid_and_provider(uid, provider) - identity = ::Identity.with_extern_uid(provider, uid).take - - identity && identity.user - end - end - - def save - super('LDAP') - end - - # instance methods - def find_user - find_by_uid_and_provider || find_by_email || build_new_user - end - - def find_by_uid_and_provider - self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) - end - - def changed? - gl_user.changed? || gl_user.identities.any?(&:changed?) - end - - def block_after_signup? - ldap_config.block_auto_created_users - end - - def allowed? - Gitlab::LDAP::Access.allowed?(gl_user) - end - - def ldap_config - Gitlab::LDAP::Config.new(auth_hash.provider) - end - - def auth_hash=(auth_hash) - @auth_hash = Gitlab::LDAP::AuthHash.new(auth_hash) - end - end - end -end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..d9d5f90596f 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,90 +1,19 @@ module Gitlab module Middleware class ReadOnly - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze API_VERSIONS = (3..4) + def self.internal_routes + @internal_routes ||= + API_VERSIONS.map { |version| "api/v#{version}/internal" } + end + def initialize(app) @app = app - @whitelisted = internal_routes end def call(env) - @env = env - @route_hash = nil - - if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') - error_message = 'You cannot do writing operations on a read-only GitLab instance' - - if json_request? - return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] - else - rack_flash.alert = error_message - rack_session['flash'] = rack_flash.to_session_value - - return [301, { 'Location' => last_visited_url }, []] - end - end - - @app.call(env) - end - - private - - def internal_routes - API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } - end - - def disallowed_request? - DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes - end - - def json_request? - request.media_type == APPLICATION_JSON - end - - def rack_flash - @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) - end - - def rack_session - @env['rack.session'] - end - - def request - @env['rack.request'] ||= Rack::Request.new(@env) - end - - def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url - end - - def route_hash - @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} - end - - def whitelisted_routes - grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - - def grack_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') - - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' - end - - def lfs_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') - - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb new file mode 100644 index 00000000000..45b644e6510 --- /dev/null +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -0,0 +1,86 @@ +module Gitlab + module Middleware + class ReadOnly + class Controller + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + + def initialize(app, env) + @app = app + @env = env + end + + def call + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + + if json_request? + return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] + else + rack_flash.alert = ERROR_MESSAGE + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(@env) + end + + private + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && + !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end + end +end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb new file mode 100644 index 00000000000..f8d0a135965 --- /dev/null +++ b/lib/gitlab/middleware/release_env.rb @@ -0,0 +1,14 @@ +module Gitlab + module Middleware + # Some of middleware would hold env for no good reason even after the + # request had already been processed, and we could not garbage collect + # them due to this. Put this middleware as the first middleware so that + # it would clear the env after the request is done, allowing GC gets a + # chance to release memory for the last request. + ReleaseEnv = Struct.new(:app) do + def call(env) + app.call(env).tap { env.clear } + end + end + end +end diff --git a/lib/gitlab/o_auth.rb b/lib/gitlab/o_auth.rb deleted file mode 100644 index 5ad8d83bd6e..00000000000 --- a/lib/gitlab/o_auth.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module OAuth - SignupDisabledError = Class.new(StandardError) - SigninDisabledForProviderError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb deleted file mode 100644 index 5b5ed449f94..00000000000 --- a/lib/gitlab/o_auth/auth_hash.rb +++ /dev/null @@ -1,90 +0,0 @@ -# Class to parse and transform the info provided by omniauth -# -module Gitlab - module OAuth - class AuthHash - attr_reader :auth_hash - def initialize(auth_hash) - @auth_hash = auth_hash - end - - def uid - @uid ||= Gitlab::Utils.force_utf8(auth_hash.uid.to_s) - end - - def provider - @provider ||= auth_hash.provider.to_s - end - - def name - @name ||= get_info(:name) || "#{get_info(:first_name)} #{get_info(:last_name)}" - end - - def username - @username ||= username_and_email[:username].to_s - end - - def email - @email ||= username_and_email[:email].to_s - end - - def password - @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) - end - - def location - location = get_info(:address) - if location.is_a?(Hash) - [location.locality.presence, location.country.presence].compact.join(', ') - else - location - end - end - - def has_attribute?(attribute) - if attribute == :location - get_info(:address).present? - else - get_info(attribute).present? - end - end - - private - - def info - auth_hash.info - end - - def get_info(key) - value = info[key] - Gitlab::Utils.force_utf8(value) if value - value - end - - def username_and_email - @username_and_email ||= begin - username = get_info(:username).presence || get_info(:nickname).presence - email = get_info(:email).presence - - username ||= generate_username(email) if email - email ||= generate_temporarily_email(username) if username - - { - username: username, - email: email - } - end - end - - # Get the first part of the email address (before @) - # In addtion in removes illegal characters - def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s - end - - def generate_temporarily_email(username) - "temp-email-for-oauth-#{username}@gitlab.localhost" - end - end - end -end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb deleted file mode 100644 index 657db29c85a..00000000000 --- a/lib/gitlab/o_auth/provider.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Gitlab - module OAuth - class Provider - LABELS = { - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google" - }.freeze - - def self.providers - Devise.omniauth_providers - end - - def self.enabled?(name) - providers.include?(name.to_sym) - end - - def self.ldap_provider?(name) - name.to_s.start_with?('ldap') - end - - def self.sync_profile_from_provider?(provider) - return true if ldap_provider?(provider) - - providers = Gitlab.config.omniauth.sync_profile_from_provider - - if providers.is_a?(Array) - providers.include?(provider) - else - providers - end - end - - def self.config_for(name) - name = name.to_s - if ldap_provider?(name) - if Gitlab::LDAP::Config.valid_provider?(name) - Gitlab::LDAP::Config.new(name).options - else - nil - end - else - Gitlab.config.omniauth.providers.find { |provider| provider.name == name } - end - end - - def self.label_for(name) - name = name.to_s - config = config_for(name) - (config && config['label']) || LABELS[name] || name.titleize - end - end - end -end diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb deleted file mode 100644 index 30739f2a2c5..00000000000 --- a/lib/gitlab/o_auth/session.rb +++ /dev/null @@ -1,19 +0,0 @@ -# :nocov: -module Gitlab - module OAuth - module Session - def self.create(provider, ticket) - Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) - end - - def self.destroy(provider, ticket) - Rails.cache.delete("gitlab:#{provider}:#{ticket}") - end - - def self.valid?(provider, ticket) - Rails.cache.read("gitlab:#{provider}:#{ticket}").present? - end - end - end -end -# :nocov: diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb deleted file mode 100644 index 28ebac1776e..00000000000 --- a/lib/gitlab/o_auth/user.rb +++ /dev/null @@ -1,241 +0,0 @@ -# OAuth extension for User model -# -# * Find GitLab user based on omniauth uid and provider -# * Create new user from omniauth data -# -module Gitlab - module OAuth - class User - attr_accessor :auth_hash, :gl_user - - def initialize(auth_hash) - self.auth_hash = auth_hash - update_profile - add_or_update_user_identities - end - - def persisted? - gl_user.try(:persisted?) - end - - def new? - !persisted? - end - - def valid? - gl_user.try(:valid?) - end - - def save(provider = 'OAuth') - raise SigninDisabledForProviderError if oauth_provider_disabled? - raise SignupDisabledError unless gl_user - - block_after_save = needs_blocking? - - Users::UpdateService.new(gl_user, user: gl_user).execute! - - gl_user.block if block_after_save - - log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" - gl_user - rescue ActiveRecord::RecordInvalid => e - log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" - return self, e.record.errors - end - - def gl_user - return @gl_user if defined?(@gl_user) - - @gl_user = find_user - end - - def find_user - user = find_by_uid_and_provider - - user ||= find_or_build_ldap_user if auto_link_ldap_user? - user ||= build_new_user if signup_enabled? - - user.external = true if external_provider? && user&.new_record? - - user - end - - protected - - def add_or_update_user_identities - return unless gl_user - - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - - identity ||= gl_user.identities.build(provider: auth_hash.provider) - identity.extern_uid = auth_hash.uid - - if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person - log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." - gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) - end - end - - def find_or_build_ldap_user - return unless ldap_person - - user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) - if user - log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - return user - end - - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - build_new_user - end - - def find_by_email - return unless auth_hash.has_attribute?(:email) - - ::User.find_by(email: auth_hash.email.downcase) - end - - def auto_link_ldap_user? - Gitlab.config.omniauth.auto_link_ldap_user - end - - def creating_linked_ldap_user? - auto_link_ldap_user? && ldap_person - end - - def ldap_person - return @ldap_person if defined?(@ldap_person) - - # Look for a corresponding person with same uid in any of the configured LDAP providers - Gitlab::LDAP::Config.providers.each do |provider| - adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = find_ldap_person(auth_hash, adapter) - break if @ldap_person - end - @ldap_person - end - - def find_ldap_person(auth_hash, adapter) - Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || - Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || - Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) - end - - def ldap_config - Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person - end - - def needs_blocking? - new? && block_after_signup? - end - - def signup_enabled? - providers = Gitlab.config.omniauth.allow_single_sign_on - if providers.is_a?(Array) - providers.include?(auth_hash.provider) - else - providers - end - end - - def external_provider? - Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) - end - - def block_after_signup? - if creating_linked_ldap_user? - ldap_config.block_auto_created_users - else - Gitlab.config.omniauth.block_auto_created_users - end - end - - def auth_hash=(auth_hash) - @auth_hash = AuthHash.new(auth_hash) - end - - def find_by_uid_and_provider - identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take - identity && identity.user - end - - def build_new_user - user_params = user_attributes.merge(skip_confirmation: true) - Users::BuildService.new(nil, user_params).execute(skip_authorization: true) - end - - def user_attributes - # Give preference to LDAP for sensitive information when creating a linked account - if creating_linked_ldap_user? - username = ldap_person.username.presence - email = ldap_person.email.first.presence - end - - username ||= auth_hash.username - email ||= auth_hash.email - - valid_username = ::Namespace.clean_path(username) - - uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } - - name = auth_hash.name - name = valid_username if name.strip.empty? - - { - name: name, - username: valid_username, - email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, - password_automatically_set: true - } - end - - def sync_profile_from_provider? - Gitlab::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) - end - - def update_profile - clear_user_synced_attributes_metadata - - return unless sync_profile_from_provider? || creating_linked_ldap_user? - - metadata = gl_user.build_user_synced_attributes_metadata - - if sync_profile_from_provider? - UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| - if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) - gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend - metadata.set_attribute_synced(key, true) - else - metadata.set_attribute_synced(key, false) - end - end - - metadata.provider = auth_hash.provider - end - - if creating_linked_ldap_user? && gl_user.email == ldap_person.email.first - metadata.set_attribute_synced(:email, true) - metadata.provider = ldap_person.provider - end - end - - def clear_user_synced_attributes_metadata - gl_user&.user_synced_attributes_metadata&.destroy - end - - def log - Gitlab::AppLogger - end - - def oauth_provider_disabled? - Gitlab::CurrentSettings.current_application_settings - .disabled_oauth_sign_in_sources - .include?(auth_hash.provider) - end - end - end -end diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/plugin.rb new file mode 100644 index 00000000000..0d1cb16b378 --- /dev/null +++ b/lib/gitlab/plugin.rb @@ -0,0 +1,26 @@ +module Gitlab + module Plugin + def self.files + Dir.glob(Rails.root.join('plugins/*')).select do |entry| + File.file?(entry) + end + end + + def self.execute_all_async(data) + args = files.map { |file| [file, data] } + + PluginWorker.bulk_perform_async(args) + end + + def self.execute(file, data) + result = Gitlab::Popen.popen_with_detail([file]) do |stdin| + stdin.write(data.to_json) + end + + exit_status = result.status&.exitstatus + [exit_status.zero?, result.stderr] + rescue => e + [false, e.message] + end + end +end diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/plugin_logger.rb new file mode 100644 index 00000000000..c4f6ec3e21d --- /dev/null +++ b/lib/gitlab/plugin_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class PluginLogger < Gitlab::Logger + def self.file_name_noext + 'plugin' + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index cf0935dbd9a..29277ec6481 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,8 +29,18 @@ module Gitlab @blobs_count ||= blobs.count end - def notes_count - @notes_count ||= notes.count + def limited_notes_count + return @limited_notes_count if defined?(@limited_notes_count) + + types = %w(issue merge_request commit snippet) + @limited_notes_count = 0 + + types.each do |type| + @limited_notes_count += notes_finder(type).limit(count_limit).count + break if @limited_notes_count >= count_limit + end + + @limited_notes_count end def wiki_blobs_count @@ -72,11 +82,12 @@ module Gitlab end def single_commit_result? - commits_count == 1 && total_result_count == 1 - end + return false if commits_count != 1 - def total_result_count - issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + counts = %i(limited_milestones_count limited_notes_count + limited_merge_requests_count limited_issues_count + blobs_count wiki_blobs_count) + counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend end private @@ -106,7 +117,11 @@ module Gitlab end def notes - @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') + @notes ||= notes_finder(nil) + end + + def notes_finder(type) + NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb deleted file mode 100644 index 33d19373098..00000000000 --- a/lib/gitlab/saml/auth_hash.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Gitlab - module Saml - class AuthHash < Gitlab::OAuth::AuthHash - def groups - Array.wrap(get_raw(Gitlab::Saml::Config.groups)) - end - - private - - def get_raw(key) - # Needs to call `all` because of https://git.io/vVo4u - # otherwise just the first value is returned - auth_hash.extra[:raw_info].all[key] - end - end - end -end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb deleted file mode 100644 index 574c3a4b28c..00000000000 --- a/lib/gitlab/saml/config.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Gitlab - module Saml - class Config - class << self - def options - Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } - end - - def groups - options[:groups_attribute] - end - - def external_groups - options[:external_groups] - end - end - end - end -end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb deleted file mode 100644 index d8faf7aad8c..00000000000 --- a/lib/gitlab/saml/user.rb +++ /dev/null @@ -1,50 +0,0 @@ -# SAML extension for User model -# -# * Find GitLab user based on SAML uid and provider -# * Create new user from SAML data -# -module Gitlab - module Saml - class User < Gitlab::OAuth::User - def save - super('SAML') - end - - def find_user - user = find_by_uid_and_provider - - user ||= find_by_email if auto_link_saml_user? - user ||= find_or_build_ldap_user if auto_link_ldap_user? - user ||= build_new_user if signup_enabled? - - if external_users_enabled? && user - # Check if there is overlap between the user's groups and the external groups - # setting then set user as external or internal. - user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - end - - user - end - - def changed? - return true unless gl_user - - gl_user.changed? || gl_user.identities.any?(&:changed?) - end - - protected - - def auto_link_saml_user? - Gitlab.config.omniauth.auto_link_saml_user - end - - def external_users_enabled? - !Gitlab::Saml::Config.external_groups.nil? - end - - def auth_hash=(auth_hash) - @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) - end - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 5a5ae7f19d4..757ef71b95a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,6 +1,8 @@ module Gitlab class SearchResults class FoundBlob + include EncodingHelper + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id def initialize(opts = {}) @@ -9,7 +11,7 @@ module Gitlab @basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) - @data = opts.fetch(:data, nil) + @data = encode_utf8(opts.fetch(:data, nil)) @per_page = opts.fetch(:per_page, 20) @project_id = opts.fetch(:project_id, nil) end @@ -60,22 +62,6 @@ module Gitlab without_count ? collection.without_count : collection end - def projects_count - @projects_count ||= projects.count - end - - def issues_count - @issues_count ||= issues.count - end - - def merge_requests_count - @merge_requests_count ||= merge_requests.count - end - - def milestones_count - @milestones_count ||= milestones.count - end - def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index b89ae2505c9..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - def call(worker, job, queue) - yield - - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - return unless MUTEX.try_lock - - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - Process.kill(signal, pid) - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb new file mode 100644 index 00000000000..c2b8d6de66e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -0,0 +1,133 @@ +require 'mutex_m' + +module Gitlab + module SidekiqMiddleware + class Shutdown + extend Mutex_m + + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # This exception can be used to request that the middleware start shutting down Sidekiq + WantShutdown = Class.new(StandardError) + + ShutdownWithoutRaise = Class.new(WantShutdown) + private_constant :ShutdownWithoutRaise + + # For testing only, to avoid race conditions (?) in Rspec mocks. + attr_reader :trace + + # We store the shutdown thread in a class variable to ensure that there + # can be only one shutdown thread in the process. + def self.create_shutdown_thread + mu_synchronize do + return unless @shutdown_thread.nil? + + @shutdown_thread = Thread.new { yield } + end + end + + # For testing only: so we can wait for the shutdown thread to finish. + def self.shutdown_thread + mu_synchronize { @shutdown_thread } + end + + # For testing only: so that we can reset the global state before each test. + def self.clear_shutdown_thread + mu_synchronize { @shutdown_thread = nil } + end + + def initialize + @trace = Queue.new if Rails.env.test? + end + + def call(worker, job, queue) + shutdown_exception = nil + + begin + yield + check_rss! + rescue WantShutdown => ex + shutdown_exception = ex + end + + return unless shutdown_exception + + self.class.create_shutdown_thread do + do_shutdown(worker, job, shutdown_exception) + end + + raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) + end + + private + + def do_shutdown(worker, job, shutdown_exception) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ + "#{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + + def check_rss! + return unless MAX_RSS > 0 + + current_rss = get_rss + return unless current_rss > MAX_RSS + + raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + kill(signal, pid) + end + + def pid + Process.pid + end + + def sleep(time) + if Rails.env.test? + @trace << [:sleep, time] + else + Kernel.sleep(time) + end + end + + def kill(signal, pid) + if Rails.env.test? + @trace << [:kill, signal, pid] + else + Process.kill(signal, pid) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index cc3c9a50555..466554e398c 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -31,10 +31,11 @@ module Gitlab raise NotImplementedError end - attr_accessor :project, :current_user, :params + attr_accessor :project, :current_user, :params, :chat_name - def initialize(project, user, params = {}) - @project, @current_user, @params = project, user, params.dup + def initialize(project, chat_name, params = {}) + @project, @current_user, @params = project, chat_name.user, params.dup + @chat_name = chat_name end private diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index a78408b0519..85aaa6b0eba 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -13,12 +13,13 @@ module Gitlab if command if command.allowed?(project, current_user) - command.new(project, current_user, params).execute(match) + command.new(project, chat_name, params).execute(match) else Gitlab::SlashCommands::Presenters::Access.new.access_denied end else - Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + Gitlab::SlashCommands::Help.new(project, chat_name, params) + .execute(available_commands, params[:text]) end end diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb new file mode 100644 index 00000000000..9a2219b7d77 --- /dev/null +++ b/lib/gitlab/string_placeholder_replacer.rb @@ -0,0 +1,27 @@ +module Gitlab + class StringPlaceholderReplacer + # This method accepts the following paras + # - string: the string to be analyzed + # - placeholder_regex: i.e. /%{project_path|project_id|default_branch|commit_sha}/ + # - block: this block will be called with each placeholder found in the string using + # the placeholder regex. If the result of the block is nil, the original + # placeholder will be returned. + + def self.replace_string_placeholders(string, placeholder_regex = nil, &block) + return string if string.blank? || placeholder_regex.blank? || !block_given? + + replace_placeholders(string, placeholder_regex, &block) + end + + class << self + private + + # If the result of the block is nil, then the placeholder is returned + def replace_placeholders(string, placeholder_regex, &block) + string.gsub(/%{(#{placeholder_regex})}/) do |arg| + yield($~[1]) || arg + end + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index f9faa134206..c6ad997a4d4 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -14,7 +14,7 @@ module Gitlab end def mark(marker_ranges) - return rich_line unless marker_ranges + return rich_line unless marker_ranges&.any? if html_escaped rich_marker_ranges = [] diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index 7ebf1c0428c..b19aa6dea35 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,13 +1,15 @@ module Gitlab class StringRegexMarker < StringRangeMarker def mark(regex, group: 0, &block) - regex_match = raw_line.match(regex) - return rich_line unless regex_match + ranges = [] - begin_index, end_index = regex_match.offset(group) - name_range = begin_index..(end_index - 1) + raw_line.scan(regex) do + begin_index, end_index = Regexp.last_match.offset(group) - super([name_range], &block) + ranges << (begin_index..(end_index - 1)) + end + + super(ranges, &block) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 9d13d1d781f..37d3512990e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,6 +9,7 @@ module Gitlab license_usage_data.merge(system_usage_data) .merge(features_usage_data) .merge(components_usage_data) + .merge(cycle_analytics_usage_data) end def to_json(force_refresh: false) @@ -71,6 +72,10 @@ module Gitlab } end + def cycle_analytics_usage_data + Gitlab::CycleAnalytics::UsageData.new.to_json + end + def features_usage_data features_usage_data_ce end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index ff4dc29efea..91b8bb2a83f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -31,7 +31,7 @@ module Gitlab return false unless can_access_git? if user.requires_ldap_check? && user.try_obtain_ldap_lease - return false unless Gitlab::LDAP::Access.allowed?(user) + return false unless Gitlab::Auth::LDAP::Access.allowed?(user) end true diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb new file mode 100644 index 00000000000..1ef369a4b67 --- /dev/null +++ b/lib/gitlab/verify/batch_verifier.rb @@ -0,0 +1,64 @@ +module Gitlab + module Verify + class BatchVerifier + attr_reader :batch_size, :start, :finish + + def initialize(batch_size:, start: nil, finish: nil) + @batch_size = batch_size + @start = start + @finish = finish + end + + # Yields a Range of IDs and a Hash of failed verifications (object => error) + def run_batches(&blk) + relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches + range = relation.first.id..relation.last.id + failures = run_batch(relation) + + yield(range, failures) + end + end + + def name + raise NotImplementedError.new + end + + def describe(_object) + raise NotImplementedError.new + end + + private + + def run_batch(relation) + relation.map { |upload| verify(upload) }.compact.to_h + end + + def verify(object) + expected = expected_checksum(object) + actual = actual_checksum(object) + + raise 'Checksum missing' unless expected.present? + raise 'Checksum mismatch' unless expected == actual + + nil + rescue => err + [object, err] + end + + # This should return an ActiveRecord::Relation suitable for calling #in_batches on + def relation + raise NotImplementedError.new + end + + # The checksum we expect the object to have + def expected_checksum(_object) + raise NotImplementedError.new + end + + # The freshly-recalculated checksum of the object + def actual_checksum(_object) + raise NotImplementedError.new + end + end + end +end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb new file mode 100644 index 00000000000..fe51edbdeeb --- /dev/null +++ b/lib/gitlab/verify/lfs_objects.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class LfsObjects < BatchVerifier + def name + 'LFS objects' + end + + def describe(object) + "LFS object: #{object.oid}" + end + + private + + def relation + LfsObject.all + end + + def expected_checksum(lfs_object) + lfs_object.oid + end + + def actual_checksum(lfs_object) + LfsObject.calculate_oid(lfs_object.file.path) + end + end + end +end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb new file mode 100644 index 00000000000..dd138e6b92b --- /dev/null +++ b/lib/gitlab/verify/rake_task.rb @@ -0,0 +1,53 @@ +module Gitlab + module Verify + class RakeTask + def self.run!(verify_kls) + verifier = verify_kls.new( + batch_size: ENV.fetch('BATCH', 200).to_i, + start: ENV['ID_FROM'], + finish: ENV['ID_TO'] + ) + + verbose = Gitlab::Utils.to_boolean(ENV['VERBOSE']) + + new(verifier, verbose).run! + end + + attr_reader :verifier, :output + + def initialize(verifier, verbose) + @verifier = verifier + @verbose = verbose + end + + def run! + say "Checking integrity of #{verifier.name}" + + verifier.run_batches { |*args| run_batch(*args) } + + say 'Done!' + end + + def verbose? + !!@verbose + end + + private + + def say(text) + puts(text) # rubocop:disable Rails/Output + end + + def run_batch(range, failures) + status_color = failures.empty? ? :green : :red + say "- #{range}: Failures: #{failures.count}".color(status_color) + + return unless verbose? + + failures.each do |object, error| + say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + end + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb new file mode 100644 index 00000000000..6972e517ea5 --- /dev/null +++ b/lib/gitlab/verify/uploads.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class Uploads < BatchVerifier + def name + 'Uploads' + end + + def describe(object) + "Upload: #{object.id}" + end + + private + + def relation + Upload.all + end + + def expected_checksum(upload) + upload.checksum + end + + def actual_checksum(upload) + Upload.hexdigest(upload.absolute_path) + end + end + end +end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index f5485eb89fa..4f776330e80 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -12,6 +12,12 @@ unless Rails.env.production? record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') end + + def visit_tag(node) + return unless node.tag_name == 'script' + + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') + end end end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index e05a3aad824..2403f57f05a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -336,7 +336,7 @@ namespace :gitlab do warn_user_is_not_gitlab start_checking "LDAP" - if Gitlab::LDAP::Config.enabled? + if Gitlab::Auth::LDAP::Config.enabled? check_ldap(args.limit) else puts 'LDAP is disabled in config/gitlab.yml' @@ -346,13 +346,13 @@ namespace :gitlab do end def check_ldap(limit) - servers = Gitlab::LDAP::Config.providers + servers = Gitlab::Auth::LDAP::Config.providers servers.each do |server| puts "Server: #{server}" begin - Gitlab::LDAP::Adapter.open(server) do |adapter| + Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| check_ldap_auth(adapter) puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 5a53eac0897..2453079911d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -87,7 +87,7 @@ namespace :gitlab do print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." - if Gitlab::LDAP::Access.allowed?(user) + if Gitlab::Auth::LDAP::Access.allowed?(user) puts " [OK]".color(:green) else if block_flag diff --git a/lib/tasks/gitlab/lfs/check.rake b/lib/tasks/gitlab/lfs/check.rake new file mode 100644 index 00000000000..869463d4e5d --- /dev/null +++ b/lib/tasks/gitlab/lfs/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :lfs do + desc 'GitLab | LFS | Check integrity of uploaded LFS objects' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::LfsObjects) + end + end +end diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake deleted file mode 100644 index df31567ce64..00000000000 --- a/lib/tasks/gitlab/uploads.rake +++ /dev/null @@ -1,44 +0,0 @@ -namespace :gitlab do - namespace :uploads do - desc 'GitLab | Uploads | Check integrity of uploaded files' - task check: :environment do - puts 'Checking integrity of uploaded files' - - uploads_batches do |batch| - batch.each do |upload| - puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) - - if upload.exist? - check_checksum(upload) - else - puts " * File does not exist on the file system".color(:red) - end - end - end - - puts 'Done!' - end - - def batch_size - ENV.fetch('BATCH', 200).to_i - end - - def calculate_checksum(absolute_path) - Digest::SHA256.file(absolute_path).hexdigest - end - - def check_checksum(upload) - checksum = calculate_checksum(upload.absolute_path) - - if checksum != upload.checksum - puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) - end - end - - def uploads_batches(&block) - Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches - yield relation - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/check.rake b/lib/tasks/gitlab/uploads/check.rake new file mode 100644 index 00000000000..2be2ec7f9c9 --- /dev/null +++ b/lib/tasks/gitlab/uploads/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::Uploads) + end + end +end diff --git a/lib/tasks/plugins.rake b/lib/tasks/plugins.rake new file mode 100644 index 00000000000..e73dd7e68df --- /dev/null +++ b/lib/tasks/plugins.rake @@ -0,0 +1,16 @@ +namespace :plugins do + desc 'Validate existing plugins' + task validate: :environment do + puts 'Validating plugins from /plugins directory' + + Gitlab::Plugin.files.each do |file| + success, message = Gitlab::Plugin.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) + + if success + puts "* #{file} succeed (zero exit code)." + else + puts "* #{file} failure (non-zero exit code). #{message}" + end + end + end +end |