diff options
author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-07-10 13:40:11 +0200 |
---|---|---|
committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-07-10 13:40:11 +0200 |
commit | 6120dc261ab3700b7a19f683559494319b6a1b75 (patch) | |
tree | 9b9c9b0c5664f03af9732ebf3c0d01079533df94 /lib | |
parent | 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 (diff) | |
parent | cd735170d3bc5e71d46ba1e37249ae713b7842f8 (diff) | |
download | gitlab-ce-6120dc261ab3700b7a19f683559494319b6a1b75.tar.gz |
Merge branch 'master' into backstage/gb/migrate-stages-statuses
* master: (587 commits)
Increase GitLab QA screenshots size
Fix QA test scenario for creating a new project
Configure headless chrome screenshots correctly
Allow passing exceptions when creating project services
Also inject new route helpers into includers of GitlabRoutingHelper
Remove many N+1 queries with merge requests API
fix milestones finder failing spec
Make wrong migrations idempotent
Remove double border on last group row
Update tests to test what the component renders and what the clicks calls
Username and password are no longer stripped from import url on import
Change `sign_out` usage to `gitlab_sign_out` in one spec
Send Gitaly Repository with /api/internal/allowed
Resolve "Liberation Mono weird font rendering on Fedora/openSUSE/other Linux distros"
Force mobile view for admin projects
fix spec
Mark a subgroup-related spec as PostgreSQL-only
Use vue .prevent to prevent the click
Add members-list class to admin projects page
Fix another typo in SHA attribute spec
...
Conflicts:
db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb
db/schema.rb
lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
spec/migrations/migrate_stage_id_reference_in_background_spec.rb
Diffstat (limited to 'lib')
82 files changed, 935 insertions, 412 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index d767af36e8e..efcf0976a81 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,6 +2,8 @@ module API class API < Grape::API include APIGuard + allow_access_with_scope :api + version %w(v3 v4), using: :path version 'v3', using: :path do @@ -44,7 +46,6 @@ module API mount ::API::V3::Variables end - before { allow_access_with_scope :api } before { header['X-Frame-Options'] = 'SAMEORIGIN' } before { Gitlab::I18n.locale = current_user&.preferred_language } diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9fcf04efa38..0d2d71e336a 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -23,6 +23,23 @@ module API install_error_responders(base) end + class_methods do + # Set the authorization scope(s) allowed for an API endpoint. + # + # A call to this method maps the given scope(s) to the current API + # endpoint class. If this method is called multiple times on the same class, + # the scopes are all aggregated. + def allow_access_with_scope(scopes, options = {}) + Array(scopes).each do |scope| + allowed_scopes << Scope.new(scope, options) + end + end + + def allowed_scopes + @scopes ||= [] + end + end + # Helper Methods for Grape Endpoint module HelperMethods # Invokes the doorkeeper guard. @@ -47,7 +64,7 @@ module API access_token = find_access_token return nil unless access_token - case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) @@ -74,18 +91,6 @@ module API @current_user end - # Set the authorization scope(s) allowed for the current request. - # - # Note: A call to this method adds to any previous scopes in place. This is done because - # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then - # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the - # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they - # need to be stored. - def allow_access_with_scope(*scopes) - @scopes ||= [] - @scopes.concat(scopes.map(&:to_s)) - end - private def find_user_by_authentication_token(token_string) @@ -96,7 +101,7 @@ module API access_token = PersonalAccessToken.active.find_by_token(token_string) return unless access_token - if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes) User.find(access_token.user_id) end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index c6fc17cc391..bcb842b9211 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -67,7 +67,7 @@ module API result = ::Files::MultiService.new(user_project, current_user, attrs).execute if result[:status] == :success - commit_detail = user_project.repository.commits(result[:result], limit: 1).first + commit_detail = user_project.repository.commit(result[:result]) present commit_detail, with: Entities::RepoCommitDetail else render_api_error!(result[:message], 400) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cef5a0abe12..f4796f311a5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -112,6 +112,7 @@ module API expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs + expose :ci_config_path expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end @@ -254,7 +255,7 @@ module API class ProjectEntity < Grape::Entity expose :id, :iid - expose(:project_id) { |entity| entity.project.id } + expose(:project_id) { |entity| entity&.project.try(:id) } expose :title, :description expose :state, :created_at, :updated_at end @@ -266,7 +267,12 @@ module API expose :deleted_file?, as: :deleted_file end - class Milestone < ProjectEntity + class Milestone < Grape::Entity + expose :id, :iid + expose(:project_id) { |entity| entity&.project_id } + expose(:group_id) { |entity| entity&.group_id } + expose :title, :description + expose :state, :created_at, :updated_at expose :due_date expose :start_date end @@ -434,7 +440,7 @@ module API target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? - Gitlab::Application.routes.url_helpers.public_send(target_url, + Gitlab::Routing.url_helpers.public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) end @@ -502,12 +508,20 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:current_user].id) + if options.key?(:project_members) + (options[:project_members] || []).find { |member| member.source_id == project.id } + else + project.project_members.find_by(user_id: options[:current_user].id) + end end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - project.group.group_members.find_by(user_id: options[:current_user].id) + if options.key?(:group_members) + (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } + else + project.group.group_members.find_by(user_id: options[:current_user].id) + end end end end @@ -831,7 +845,7 @@ module API end class Cache < Grape::Entity - expose :key, :untracked, :paths + expose :key, :untracked, :paths, :policy end class Credentials < Grape::Entity @@ -874,5 +888,11 @@ module API expose :dependencies, using: Dependency end end + + class UserAgentDetail < Grape::Entity + expose :user_agent + expose :ip_address + expose :submitted, as: :akismet_submitted + end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 21745916463..9385c6ca174 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -14,14 +14,12 @@ module API end end - def gate_target(params) - if params[:feature_group] - Feature.group(params[:feature_group]) - elsif params[:user] - User.find_by_username(params[:user]) - else - gate_value(params) - end + def gate_targets(params) + targets = [] + targets << Feature.group(params[:feature_group]) if params[:feature_group] + targets << User.find_by_username(params[:user]) if params[:user] + + targets end end @@ -42,18 +40,25 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' - mutually_exclusive :feature_group, :user end post ':name' do feature = Feature.get(params[:name]) - target = gate_target(params) + targets = gate_targets(params) value = gate_value(params) case value when true - feature.enable(target) + if targets.present? + targets.each { |target| feature.enable(target) } + else + feature.enable + end when false - feature.disable(target) + if targets.present? + targets.each { |target| feature.disable(target) } + else + feature.disable + end else feature.enable_percentage_of_time(value) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2c73a6fdc4e..0f4791841d2 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -268,6 +268,7 @@ module API finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] finder_params[:archived] = params[:archived] finder_params[:search] = params[:search] if params[:search] + finder_params[:user] = params.delete(:user) if params[:user] finder_params end @@ -313,7 +314,7 @@ module API def present_artifacts!(artifacts_file) return not_found! unless artifacts_file.exists? - + if artifacts_file.file_storage? present_file!(artifacts_file.path, artifacts_file.filename) else @@ -342,8 +343,8 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: @scopes) - @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) + @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) @initial_current_user ||= find_user_from_warden unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? @@ -407,5 +408,22 @@ module API exception.status == 500 end + + # An array of scopes that were registered (using `allow_access_with_scope`) + # for the current endpoint class. It also returns scopes registered on + # `API::API`, since these are meant to apply to all API routes. + def scopes_registered_for_endpoint + @scopes_registered_for_endpoint ||= + begin + endpoint_classes = [options[:for].presence, ::API::API].compact + endpoint_classes.reduce([]) do |memo, endpoint| + if endpoint.respond_to?(:allowed_scopes) + memo.concat(endpoint.allowed_scopes) + else + memo + end + end + end + end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 5e9cf5e68b1..ecb79317093 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,6 +1,11 @@ module API module Helpers module InternalHelpers + SSH_GITALY_FEATURES = { + 'git-receive-pack' => :ssh_receive_pack, + 'git-upload-pack' => :ssh_upload_pack + }.freeze + def wiki? set_project unless defined?(@wiki) @wiki @@ -10,7 +15,7 @@ module API set_project unless defined?(@project) @project end - + def redirected_path @redirected_path end @@ -54,15 +59,33 @@ module API Gitlab::GlRepository.gl_repository(project, wiki?) end - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - def repository_path + # Return the repository depending on whether we want the wiki or the + # regular repository + def repository if wiki? - project.wiki.repository.path_to_repo + project.wiki.repository else - project.repository.path_to_repo + project.repository end end + + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + def repository_path + repository.path_to_repo + end + + # Return the Gitaly Address if it is enabled + def gitaly_payload(action) + feature = SSH_GITALY_FEATURES[action] + return unless feature && Gitlab::GitalyClient.feature_enabled?(feature) + + { + repository: repository.gitaly_repository, + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index f1c79970ba4..ef2c08e902c 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -47,7 +47,8 @@ module API { status: true, gl_repository: gl_repository, - repository_path: repository_path + repository_path: repository_path, + gitaly: gitaly_payload(params[:action]) } end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 09dca0dff8b..64be08094ed 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -241,6 +241,22 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + + desc 'Get the user agent details for an issue' do + success Entities::UserAgentDetail + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ":id/issues/:issue_iid/user_agent_detail" do + authenticated_as_admin! + + issue = find_project_issue(params[:issue_iid]) + + return not_found!('UserAgentDetail') unless issue.user_agent_detail + + present issue.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 1118fc7465b..d419d345ec5 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -41,7 +41,9 @@ module API args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) - merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + merge_requests = MergeRequestsFinder.new(current_user, args).execute + .inc_notes_with_associations + .preload(:target_project, :author, :assignee, :milestone, :merge_request_diff) merge_requests.reorder(args[:order_by] => args[:sort]) end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 93d89209934..dbeaf9e17ef 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -74,9 +74,10 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :update_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -92,9 +93,10 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :update_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -110,9 +112,10 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :admin_pipeline_schedule, user_project + authorize! :read_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :admin_pipeline_schedule, pipeline_schedule status :accepted present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 64efe82a937..3320eadff0d 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -131,6 +131,22 @@ module API content_type 'text/plain' present snippet.content end + + desc 'Get the user agent details for a project snippet' do + success Entities::UserAgentDetail + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id/user_agent_detail" do + authenticated_as_admin! + + snippet = Snippet.find_by!(id: params[:id]) + + return not_found!('UserAgentDetail') unless snippet.user_agent_detail + + present snippet.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d0bd64b2972..c459257158d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -10,6 +10,7 @@ module API helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' @@ -35,61 +36,86 @@ module API params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end - end - resource :projects do - helpers do - params :collection_params do - use :sort_params - use :filter_params - use :pagination - - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - end + params :collection_params do + use :sort_params + use :filter_params + use :pagination - params :sort_params do - optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], - default: 'created_at', desc: 'Return projects ordered by field' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return projects sorted in ascending and descending order' - end + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end - params :filter_params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, - desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of projects matching the search criteria' - optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' - optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' - optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' - optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' - end + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end - params :create_params do - optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' - optional :import_url, type: String, desc: 'URL from which the project is imported' - end + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of projects matching the search criteria' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end - def present_projects(options = {}) - projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute - projects = reorder_projects(projects) - projects = projects.with_statistics if params[:statistics] - projects = projects.with_issues_enabled if params[:with_issues_enabled] - projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] - - options = options.reverse_merge( - with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, - statistics: params[:statistics], - current_user: current_user - ) - options[:with] = Entities::BasicProjectDetails if params[:simple] - - present paginate(projects), options + def present_projects(options = {}) + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + + if current_user + projects = projects.includes(:route, :taggings, namespace: :route) + project_members = current_user.project_members + group_members = current_user.group_members end + + options = options.reverse_merge( + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + project_members: project_members, + group_members: group_members, + current_user: current_user + ) + options[:with] = Entities::BasicProjectDetails if params[:simple] + + present paginate(projects), options end + end + resource :users, requirements: { user_id: %r{[^/]+} } do + desc 'Get a user projects' do + success Entities::BasicProjectDetails + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :collection_params + use :statistics_params + end + get ":user_id/projects" do + user = find_user(params[:user_id]) + not_found!('User') unless user + + params[:user] = user + + present_projects + end + end + + resource :projects do desc 'Get a list of visible projects for authenticated user' do success Entities::BasicProjectDetails end diff --git a/lib/api/scope.rb b/lib/api/scope.rb new file mode 100644 index 00000000000..d5165b2e482 --- /dev/null +++ b/lib/api/scope.rb @@ -0,0 +1,23 @@ +# Encapsulate a scope used for authorization, such as `api`, or `read_user` +module API + class Scope + attr_reader :name, :if + + def initialize(name, options = {}) + @name = name.to_sym + @if = options[:if] + end + + # Are the `scopes` passed in sufficient to adequately authorize the passed + # request for the scope represented by the current instance of this class? + def sufficient?(scopes, request) + scopes.include?(self.name) && verify_if_condition(request) + end + + private + + def verify_if_condition(request) + self.if.nil? || self.if.call(request) + end + end +end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c630c24c339..fd634037a77 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -140,6 +140,22 @@ module API content_type 'text/plain' present snippet.content end + + desc 'Get the user agent details for a snippet' do + success Entities::UserAgentDetail + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/user_agent_detail" do + authenticated_as_admin! + + snippet = Snippet.find_by!(id: params[:id]) + + return not_found!('UserAgentDetail') unless snippet.user_agent_detail + + present snippet.user_agent_detail, with: Entities::UserAgentDetail + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index f9555842daf..c469751c31c 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,13 +1,15 @@ module API class Users < Grape::API include PaginationParams + include APIGuard - before do - allow_access_with_scope :read_user if request.get? - authenticate! - end + allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + before do + authenticate_non_get! + end + helpers do def find_user(params) id = params[:user_id] || params[:id] @@ -46,20 +48,33 @@ module API optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + optional :created_after, type: DateTime, desc: 'Return users created after the specified time' + optional :created_before, type: DateTime, desc: 'Return users created before the specified time' all_or_none_of :extern_uid, :provider use :pagination end get do - unless can?(current_user, :read_users_list) - render_api_error!("Not authorized.", 403) - end - authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + unless current_user&.admin? + params.except!(:created_after, :created_before) + end + users = UsersFinder.new(current_user, params).execute - entity = current_user.admin? ? Entities::UserWithAdmin : Entities::UserBasic + authorized = can?(current_user, :read_users_list) + + # When `current_user` is not present, require that the `username` + # parameter is passed, to prevent an unauthenticated user from accessing + # a list of all the users on the GitLab instance. `UsersFinder` performs + # an exact match on the `username` parameter, so we are guaranteed to + # get either 0 or 1 `users` here. + authorized &&= params[:username].present? if current_user.blank? + + forbidden!("Not authorized to access /api/v4/users") unless authorized + + entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic present paginate(users), with: entity end @@ -398,6 +413,10 @@ module API end resource :user do + before do + authenticate! + end + desc 'Get the currently authenticated user' do success Entities::UserPublic end diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 37020019e07..cf106f2552d 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -2,9 +2,11 @@ module API module V3 class Users < Grape::API include PaginationParams + include APIGuard + + allow_access_with_scope :read_user, if: -> (request) { request.get? } before do - allow_access_with_scope :read_user if request.get? authenticate! end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 10374995497..7fa528fb2d3 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -45,7 +45,9 @@ module API optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do - variable = user_project.variables.create(declared_params(include_missing: false)) + variable_params = declared_params(include_missing: false) + + variable = user_project.variables.create(variable_params) if variable.valid? present variable, with: Entities::Variable @@ -67,7 +69,9 @@ module API return not_found!('Variable') unless variable - if variable.update(declared_params(include_missing: false).except(:key)) + variable_params = declared_params(include_missing: false).except(:key) + + if variable.update(variable_params) present variable, with: Entities::Variable else render_validation_error!(variable) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 8bc2dd18bda..7a262dd025c 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -216,12 +216,7 @@ module Banzai @references_per_project ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = - if uses_reference_pattern? - Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) - else - object_class.link_reference_pattern - end + regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) nodes.each do |node| node.to_html.scan(regex) do @@ -323,14 +318,6 @@ module Banzai value end end - - # There might be special cases like filters - # that should ignore reference pattern - # eg: IssueReferenceFilter when using a external issues tracker - # In those cases this method should be overridden on the filter subclass - def uses_reference_pattern? - true - end end end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index eaacb9591b1..21bcb1c5ca8 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -30,7 +30,7 @@ module Banzai def url_for_object(range, project) h = Gitlab::Routing.url_helpers - h.namespace_project_compare_url(project.namespace, project, + h.project_compare_url(project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 69c06117eda..714e0319025 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - h.namespace_project_commit_url(project.namespace, project, commit, + h.project_commit_url(project, commit, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index dce4de3ceaf..53a229256a5 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -3,6 +3,8 @@ module Banzai # HTML filter that replaces external issue tracker references with links. # References are ignored if the project doesn't use an external issue # tracker. + # + # This filter does not support cross-project references. class ExternalIssueReferenceFilter < ReferenceFilter self.reference_type = :external_issue @@ -87,7 +89,7 @@ module Banzai end def issue_reference_pattern - external_issues_cached(:issue_reference_pattern) + external_issues_cached(:external_issue_reference_pattern) end private diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 044d18ff824..ba1a5ac84b3 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -15,10 +15,6 @@ module Banzai Issue end - def uses_reference_pattern? - context[:project].default_issues_tracker? - end - def find_object(project, iid) issues_per_project[project][iid] end @@ -38,13 +34,7 @@ module Banzai projects_per_reference.each do |path, project| issue_ids = references_per_project[path] - - issues = - if project.default_issues_tracker? - project.issues.where(iid: issue_ids.to_a) - else - issue_ids.map { |id| ExternalIssue.new(id, project) } - end + issues = project.issues.where(iid: issue_ids.to_a) issues.each do |issue| hash[project][issue.iid.to_i] = issue @@ -55,26 +45,6 @@ module Banzai end end - def object_link_title(object) - if object.is_a?(ExternalIssue) - "Issue in #{object.project.external_issue_tracker.title}" - else - super - end - end - - def data_attributes_for(text, project, object, link: false) - if object.is_a?(ExternalIssue) - data_attribute( - project: project.id, - external_issue: object.id, - reference_type: ExternalIssueReferenceFilter.reference_type - ) - else - super - end - end - def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a605dea149e..5364984c9d3 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -61,8 +61,7 @@ module Banzai def url_for_object(label, project) h = Gitlab::Routing.url_helpers - h.namespace_project_issues_url(project.namespace, project, label_name: label.name, - only_path: context[:only_path]) + h.project_issues_url(project, label_name: label.name, only_path: context[:only_path]) end def object_link_text(object, matches) diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 3888acf935e..0eab865ac04 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -17,7 +17,7 @@ module Banzai def url_for_object(mr, project) h = Gitlab::Routing.url_helpers - h.namespace_project_merge_request_url(project.namespace, project, mr, + h.project_merge_request_url(project, mr, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index f12014e191f..45c033d32a8 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -49,7 +49,7 @@ module Banzai def url_for_object(milestone, project) h = Gitlab::Routing.url_helpers - h.namespace_project_milestone_url(project.namespace, project, milestone, + h.project_milestone_url(project, milestone, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 6640168bfa2..a6f8650ed3d 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -30,6 +30,8 @@ module Banzai attributes = attributes.reject { |_, v| v.nil? } attributes[:reference_type] ||= self.class.reference_type + attributes[:container] ||= 'body' + attributes[:placement] ||= 'bottom' attributes.delete(:original) if context[:no_original_data] attributes.map do |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 212a0bbf2a0..134a192c22b 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -17,7 +17,7 @@ module Banzai def url_for_object(snippet, project) h = Gitlab::Routing.url_helpers - h.namespace_project_snippet_url(project.namespace, project, snippet, + h.project_snippet_url(project, snippet, only_path: context[:only_path]) end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index a798927823f..f3356d6c51e 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -107,7 +107,7 @@ module Banzai if author && !project.team.member?(author) link_content else - url = urls.namespace_project_url(project.namespace, project, + url = urls.project_url(project, only_path: context[:only_path]) data = data_attribute(project: project.id, author: author.try(:id)) diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 9fd4bd68d43..a65bbe23958 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -4,9 +4,6 @@ module Banzai self.reference_type = :issue def nodes_visible_to_user(user, nodes) - # It is not possible to check access rights for external issue trackers - return nodes if project && project.external_issue_tracker - issues = issues_for_nodes(nodes) readable_issues = Ability diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index dd864eea3fa..721ed97bb6b 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -126,8 +126,7 @@ module ExtractsPath raise InvalidPathError unless @commit @hex_path = Digest::SHA1.hexdigest(@path) - @logs_path = logs_file_namespace_project_ref_path(@project.namespace, - @project, @ref, @path) + @logs_path = logs_file_project_ref_path(@project, @ref, @path) rescue RuntimeError, NoMethodError, InvalidPathError render_404 diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3933c3b04dd..ccb5d886bab 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -130,13 +130,13 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES) Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_scoped_token?(token, ["api"]) + token && token.accessible? && valid_scoped_token?(token, [:api]) end def valid_scoped_token?(token, scopes) diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index d1d0a968588..91540127ea9 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,15 +1,15 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference - def perform(id) + def perform(start_id, stop_id) sql = <<-SQL.strip_heredoc - UPDATE "ci_builds" - SET "stage_id" = + UPDATE ci_builds + SET stage_id = (SELECT id FROM ci_stages WHERE ci_stages.pipeline_id = ci_builds.commit_id AND ci_stages.name = ci_builds.stage) - WHERE "ci_builds"."id" = #{id.to_i} - AND "ci_builds"."stage_id" IS NULL + WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND ci_builds.stage_id IS NULL SQL ActiveRecord::Base.connection.execute(sql) diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb index f87a7b7942e..2ee35a0d4c1 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/build/metadata.rb @@ -15,12 +15,11 @@ module Gitlab end def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) + build_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, id: @ref) end end end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb index 53588185622..e898f5d790e 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -16,13 +16,11 @@ module Gitlab end def image_url - coverage_namespace_project_badges_url(@project.namespace, - @project, @ref, - format: :svg) + coverage_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, @ref) end end end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 4a049ef758d..8ad6f3cb986 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -4,7 +4,7 @@ module Gitlab # Abstract class for badge metadata # class Metadata - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing include ActionView::Helpers::AssetTagHelper include ActionView::Helpers::UrlHelper diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f074df9c7a1..d7e09acbbf3 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,11 +7,14 @@ module Gitlab # class Cache < Node include Configurable + include Attributable - ALLOWED_KEYS = %i[key untracked paths].freeze + ALLOWED_KEYS = %i[key untracked paths policy].freeze + DEFAULT_POLICY = 'pull-push'.freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true end entry :key, Entry::Key, @@ -25,8 +28,15 @@ module Gitlab helpers :key + attributes :policy + def value - super.merge(key: key_value) + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + + result end end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 897dcff8012..6555c589173 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -15,7 +15,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :name, type: String, presence: true - validates :entrypoint, type: String, allow_nil: true + validates :entrypoint, array_of_strings: true, allow_nil: true end def hash? diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index b52faf48b58..3e2ebcff31a 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,8 +15,8 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :name, type: String, presence: true - validates :entrypoint, type: String, allow_nil: true - validates :command, type: String, allow_nil: true + validates :entrypoint, array_of_strings: true, allow_nil: true + validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 439ef0ce015..8ad3e57e59d 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,9 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + cancel_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index b173c23fba4..c0c7c7f5b5d 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + project_job_path(subject.project, subject) end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index e80f3263794..c7726543599 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 56303e4cb17..8c8fdc56d75 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,9 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + retry_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 2778d6f3b52..d464738deaf 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 76bfd18bf40..61bb07beb0f 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject) + project_pipeline_path(subject.project, subject) end def has_action? diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index 7852f492e1d..bc99d925347 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -8,10 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject.pipeline, - anchor: subject.name) + project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) end def has_action? diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 75a213ef752..98dfe900044 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,7 +1,7 @@ module Gitlab module Conflict class File - include Gitlab::Routing.url_helpers + include Gitlab::Routing include IconsHelper MissingResolution = Class.new(ResolutionError) @@ -205,9 +205,7 @@ module Gitlab old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), - blob_path: namespace_project_blob_path(merge_request.project.namespace, - merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)) + blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path)) } json_hash.tap do |json_hash| @@ -223,11 +221,10 @@ module Gitlab end def content_path - conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, - merge_request.project, - merge_request, - old_path: their_path, - new_path: our_path) + conflict_for_path_project_merge_request_path(merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 9d25ef078e8..f5d08c0b658 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -13,6 +13,10 @@ module Gitlab MergeRequestDiff.arel_table end + def mr_diff_commits_table + MergeRequestDiffCommit.arel_table + end + def mr_closing_issues_table MergeRequestsClosingIssues.arel_table end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7d342a2d2cb..b260822788d 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -2,40 +2,59 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @projections = [mr_diff_table[:st_commits].as('commits'), + @projections = [mr_diff_table[:id], + mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end def events_query - base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + base_query + .join(mr_diff_table) + .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) super end private + def merge_request_diff_commits + @merge_request_diff_commits ||= + MergeRequestDiffCommit + .where(merge_request_diff_id: event_result.map { |event| event['id'] }) + .group_by(&:merge_request_diff_id) + end + def serialize(event) - st_commit = first_time_reference_commit(event.delete('commits'), event) + commit = first_time_reference_commit(event) - return unless st_commit + return unless commit - serialize_commit(event, st_commit, query) + serialize_commit(event, commit, query) end - def first_time_reference_commit(commits, event) + def first_time_reference_commit(event) + return nil unless event && merge_request_diff_commits + + commits = + if event['st_commits'].present? + YAML.load(event['st_commits']) + else + merge_request_diff_commits[event['id'].to_i] + end + return nil if commits.blank? - YAML.load(commits).find do |commit| + commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i end end - def serialize_commit(event, st_commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + def serialize_commit(event, commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 1a5887dab7e..edd7795eef0 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -337,7 +337,7 @@ module Gitlab # In the EE repo $ git push origin #{ee_branch_prefix} - ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. Once this is done, you can retry this failed build, and it should pass. diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index ea035e33eff..dd1d9dcd555 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -4,7 +4,7 @@ module Gitlab class RepositoryPush attr_reader :author_id, :ref, :action - include Gitlab::Routing.url_helpers + include Gitlab::Routing include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project @@ -96,20 +96,13 @@ module Gitlab def target_url if @action == :push && commits if commits.length > 1 - namespace_project_compare_url(project_namespace, - project, - from: compare.start_commit, - to: compare.head_commit) + project_compare_url(project, from: compare.start_commit, to: compare.head_commit) else - namespace_project_commit_url(project_namespace, - project, - commits.first) + project_commit_url(project, commits.first) end else unless @action == :delete - namespace_project_tree_url(project_namespace, - project, - ref_name) + project_tree_url(project, ref_name) end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index a7aceab4c14..ea386c2ddcb 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -41,10 +41,6 @@ module Gitlab commit_id: sha ) when :BLOB - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data) new( id: entry.oid, name: name, @@ -53,7 +49,7 @@ module Gitlab mode: entry.mode.to_s(8), path: path, commit_id: sha, - binary: detect && detect[:type] == :binary + binary: binary?(entry.data) ) end end @@ -87,14 +83,28 @@ module Gitlab end def raw(repository, sha) - blob = repository.lookup(sha) + Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + else + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + end + end - new( - id: blob.oid, - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - binary: blob.binary? - ) + def binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary end # Recursive search of blob id by path @@ -165,8 +175,17 @@ module Gitlab return if @data == '' # don't mess with submodule blobs return @data if @loaded_all_data + Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + @data = begin + if is_enabled + Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data + else + repository.lookup(id).content + end + end + end + @loaded_all_data = true - @data = repository.lookup(id).content @loaded_size = @data.bytesize @binary = nil end @@ -175,6 +194,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def truncated? size && (size > loaded_size) end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 1add037fa5f..666743006e5 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -110,10 +110,6 @@ module Gitlab if segment == '..' raise IndexError, 'Path cannot include directory traversal' end - - unless segment =~ Gitlab::Regex.file_name_regex - raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" - end end pathname.to_s diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 23d0c8a9bdb..e51966313d4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -554,22 +554,31 @@ module Gitlab # # => git@localhost:rack.git # def submodule_url_for(ref, path) - if submodules(ref).any? - submodule = submodules(ref)[path] - - if submodule - submodule['url'] + Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| + if is_enabled + gitaly_submodule_url_for(ref, path) + else + if submodules(ref).any? + submodule = submodules(ref)[path] + submodule['url'] if submodule + end end end end # Return total commits count accessible from passed ref def commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + gitaly_migrate(:commit_count) do |is_enabled| + if is_enabled + gitaly_commit_client.commit_count(ref) + else + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + end end # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or @@ -915,6 +924,18 @@ module Gitlab fill_submodule_ids(commit, parser.parse) end + def gitaly_submodule_url_for(ref, path) + # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. + commit_object = gitaly_commit_client.tree_entry(ref, path, 1) + + return unless commit_object && commit_object.type == :COMMIT + + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Blob::MAX_DATA_DISPLAY_SIZE) + found_module = GitmodulesParser.new(gitmodules.data).parse[path] + + found_module && found_module['url'] + end + def alternate_object_directories Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b9afa05c819..b6d4e6cfe46 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -80,6 +80,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def dir? type == :tree end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f605c06dfc3..197a94487eb 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -70,12 +70,8 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.enabled? - Gitlab.config.gitaly.enabled - end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) - return false if !enabled? || status == MigrationStatus::DISABLED + return false if status == MigrationStatus::DISABLED feature = Feature.get("gitaly_#{feature}") diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob.rb new file mode 100644 index 00000000000..0c398b46a08 --- /dev/null +++ b/lib/gitlab/gitaly_client/blob.rb @@ -0,0 +1,30 @@ +module Gitlab + module GitalyClient + class Blob + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + end + + def get_blob(oid:, limit:) + request = Gitaly::GetBlobRequest.new( + repository: @gitaly_repo, + oid: oid, + limit: limit + ) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + + blob = response.first + return unless blob.oid.present? + + data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + + Gitlab::Git::Blob.new( + id: blob.oid, + size: blob.size, + data: data, + binary: Gitlab::Git::Blob.binary?(data) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index b8877619797..aafc0520664 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -56,6 +56,15 @@ module Gitlab entry end + def commit_count(ref) + request = Gitaly::CountCommitsRequest.new( + repository: @gitaly_repo, + revision: ref + ) + + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index 2d61992f595..6edc69de078 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -34,7 +34,7 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - GitalyClient.call(@storage, :ref, :find_ref_name, request).name + encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup) end def count_tag_names diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 319633656ff..2d1ae6a5925 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -2,11 +2,14 @@ module Gitlab module GonHelper + include WebpackHelper + def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index e78b7f22e03..70da4080cae 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -52,7 +52,7 @@ module Gitlab ] end rescue RuntimeError => ex - Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index db7cdf4b5c7..f3d489aad0d 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -12,7 +12,8 @@ module Gitlab 'zh_HK' => '繁體中文(香港)', 'zh_TW' => '繁體中文(臺灣)', 'bg' => 'български', - 'eo' => 'Esperanto' + 'eo' => 'Esperanto', + 'it' => 'Italiano' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1860352c96d..c8ad3a7a5e0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,7 @@ project_tree: - :author - :events - merge_request_diff: + - :merge_request_diff_commits - :merge_request_diff_files - :events - :timelogs diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index c56c1a4322f..cdbdfa10d0e 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -76,5 +76,44 @@ module Gitlab url.to_s end + + def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + config = { + apiVersion: 'v1', + clusters: [ + name: 'gitlab-deploy', + cluster: { + server: url + } + ], + contexts: [ + name: 'gitlab-deploy', + context: { + cluster: 'gitlab-deploy', + namespace: namespace, + user: 'gitlab-deploy' + } + ], + 'current-context': 'gitlab-deploy', + kind: 'Config', + users: [ + { + name: 'gitlab-deploy', + user: { token: token } + } + ] + } + + kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem + + config.deep_stringify_keys + end + + private + + def kubeconfig_embed_ca_pem(config, ca_pem) + cluster = config.dig(:clusters, 0, :cluster) + cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + end end end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb new file mode 100644 index 00000000000..219accfc029 --- /dev/null +++ b/lib/gitlab/metrics/base_sampler.rb @@ -0,0 +1,94 @@ +require 'logger' +module Gitlab + module Metrics + class BaseSampler + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance + end + + attr_reader :running + + # interval - The sampling interval in seconds. + def initialize(interval) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return if running + @running = true + + @thread = Thread.new do + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + end + end + + def stop + @mutex.synchronize do + return unless running + + @running = false + + if @thread + @thread.wakeup if @thread.alive? + @thread.join + @thread = nil + end + end + end + + def safe_sample + sample + rescue => e + Rails.logger.warn("#{self.class}: #{e}, stopping") + stop + end + + def sample + raise NotImplementedError + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb new file mode 100644 index 00000000000..b3da360be8f --- /dev/null +++ b/lib/gitlab/metrics/connection_rack_middleware.rb @@ -0,0 +1,45 @@ +module Gitlab + module Metrics + class ConnectionRackMiddleware + def initialize(app) + @app = app + end + + def self.rack_request_count + @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count') + end + + def self.rack_response_count + @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count') + end + + def self.rack_execution_time + @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + ConnectionRackMiddleware.rack_request_count.increment(method: method) + + status, headers, body = @app.call(env) + + ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status) + [status, headers, body] + rescue + ConnectionRackMiddleware.rack_uncaught_errors_count.increment + raise + ensure + elapsed = Time.now.to_f - started + ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed) + end + end + end + end +end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/influx_sampler.rb index 0000450d9bb..6db1dd755b7 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/influx_sampler.rb @@ -5,14 +5,11 @@ module Gitlab # This class is used to gather statistics that can't be directly associated # with a transaction such as system memory usage, garbage collection # statistics, etc. - class Sampler + class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - interval_half = interval.to_f / 2 - - @interval = interval - @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @last_step = nil + super(interval) + @last_step = nil @metrics = [] @@ -26,18 +23,6 @@ module Gitlab end end - def start - Thread.new do - Thread.current.abort_on_exception = true - - loop do - sleep(sleep_interval) - - sample - end - end - end - def sample sample_memory_usage sample_file_descriptors @@ -86,7 +71,7 @@ module Gitlab end def sample_gc - time = GC::Profiler.total_time * 1000.0 + time = GC::Profiler.total_time * 1000.0 stats = GC.stat.merge(total_time: time) # We want the difference of GC runs compared to the last sample, not the @@ -111,23 +96,6 @@ module Gitlab def sidekiq? Sidekiq.server? end - - # Returns the sleep interval with a random adjustment. - # - # The random adjustment is put in place to ensure we: - # - # 1. Don't generate samples at the exact same interval every time (thus - # potentially missing anything that happens in between samples). - # 2. Don't sample data at the same interval two times in a row. - def sleep_interval - while step = @interval_steps.sample - if step != @last_step - @last_step = step - - return @interval + @last_step - end - end - end end end end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 9d314a56e58..fb7bbc7cfc7 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -29,8 +29,8 @@ module Gitlab provide_metric(name) || registry.summary(name, docstring, base_labels) end - def gauge(name, docstring, base_labels = {}) - provide_metric(name) || registry.gauge(name, docstring, base_labels) + def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) + provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode) end def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb new file mode 100644 index 00000000000..f6987252039 --- /dev/null +++ b/lib/gitlab/metrics/unicorn_sampler.rb @@ -0,0 +1,48 @@ +module Gitlab + module Metrics + class UnicornSampler < BaseSampler + def initialize(interval) + super(interval) + end + + def unicorn_active_connections + @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + end + + def unicorn_queued_connections + @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + end + + def enabled? + # Raindrops::Linux.tcp_listener_stats is only present on Linux + unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) + end + + def sample + Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + end + + Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + end + end + + private + + def tcp_listeners + @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) + end + + def unix_listeners + @unix_listeners ||= Unicorn.listener_names - tcp_listeners + end + + def unicorn_with_listeners? + defined?(Unicorn) && Unicorn.listener_names.any? + end + end + end +end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 0d541935bc6..22332474945 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -34,7 +34,7 @@ module Gitlab write_csv do |csv| ActiveRecord::Base.transaction do - User.with_two_factor.in_batches do |relation| + User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) rows.each do |row| user = %i[id ciphertext iv salt].zip(row).to_h diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 10eb99fb461..d81f825ef96 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -112,6 +112,7 @@ module Gitlab # this group would not be accessible through `/groups/parent/activity` since # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ + - activity analytics audit_events diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 163a40ad306..2da2ce45ebc 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,33 @@ module Gitlab module PerformanceBar - def self.enabled? - Feature.enabled?('gitlab_performance_bar') + include Gitlab::CurrentSettings + + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids'.freeze + + def self.enabled?(user = nil) + return false unless user && allowed_group_id + + allowed_user_ids.include?(user.id) + end + + def self.allowed_group_id + current_application_settings.performance_bar_allowed_group_id + end + + def self.allowed_user_ids + Rails.cache.fetch(ALLOWED_USER_IDS_KEY) do + group = Group.find_by_id(allowed_group_id) + + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end + end + end + + def self.expire_allowed_user_ids_cache + Rails.cache.delete(ALLOWED_USER_IDS_KEY) end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b706434217d..c1ee20b6977 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,14 +19,6 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze - end - - def file_name_regex_message - "can contain only letters, digits, '_', '-', '@', '+' and '.'." - end - def container_registry_reference_regex Gitlab::PathRegex.git_reference_regex end @@ -38,8 +30,12 @@ module Gitlab @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} end + def environment_name_regex_chars + 'a-zA-Z0-9_/\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze end def environment_name_regex_message diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb deleted file mode 100644 index 36d1a8a6f64..00000000000 --- a/lib/gitlab/routes/legacy_builds.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module Routes - class LegacyBuilds - def initialize(map) - @map = map - end - - def draw - @map.instance_eval do - resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do - collection do - resources :artifacts, only: [], controller: 'build_artifacts' do - collection do - get :latest_succeeded, - path: '*ref_name_and_path', - format: false - end - end - end - - member do - get :raw - end - - resource :artifacts, only: [], controller: 'build_artifacts' do - get :download - get :browse, path: 'browse(/*path)', format: false - get :file, path: 'file/*path', format: false - get :raw, path: 'raw/*path', format: false - end - end - end - end - end - end -end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 632e2d87500..e57890f1143 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -2,10 +2,35 @@ module Gitlab module Routing extend ActiveSupport::Concern + mattr_accessor :_includers + self._includers = [] + included do + Gitlab::Routing.includes_helpers(self) + include Gitlab::Routing.url_helpers end + def self.includes_helpers(klass) + self._includers << klass + end + + def self.add_helpers(mod) + url_helpers.include mod + url_helpers.extend mod + + GitlabRoutingHelper.include mod + GitlabRoutingHelper.extend mod + + app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module + app_url_helpers.include mod + app_url_helpers.extend mod + + _includers.each do |klass| + klass.include mod + end + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 22554236c38..0baea092e6a 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -2,6 +2,8 @@ require 'securerandom' module Gitlab class Shell + GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze + Error = Class.new(StandardError) KeyAdder = Struct.new(:io) do @@ -67,8 +69,8 @@ module Gitlab # add_repository("/path/to/storage", "gitlab/gitlab-ci") # def add_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'add-project', storage, "#{name}.git"]) end # Import repository @@ -82,10 +84,9 @@ module Gitlab def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. - output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) - raise Error, output unless status.zero? - true + cmd = [gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] + gitlab_shell_fast_execute_raise_error(cmd) end # Fetch remote for repository @@ -103,9 +104,7 @@ module Gitlab args << '--force' if forced args << '--no-tags' if no_tags - output, status = Popen.popen(args) - raise Error, output unless status.zero? - true + gitlab_shell_fast_execute_raise_error(args) end # Move repository @@ -117,8 +116,8 @@ module Gitlab # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # def mv_repository(storage, path, new_path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - storage, "#{path}.git", "#{new_path}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace @@ -131,9 +130,9 @@ module Gitlab # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - forked_from_storage, "#{path}.git", forked_to_storage, - fork_namespace]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', + forked_from_storage, "#{path}.git", forked_to_storage, + fork_namespace]) end # Remove repository from file system @@ -145,8 +144,8 @@ module Gitlab # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # def remove_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'rm-project', storage, "#{name}.git"]) end # Add new key to gitlab-shell @@ -155,8 +154,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, self.class.strip_key(key_content)]) + gitlab_shell_fast_execute([gitlab_shell_keys_path, + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -175,8 +174,10 @@ module Gitlab # remove_key("key-342", "sha-rsa ...") # def remove_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'rm-key', key_id, key_content]) + args = [gitlab_shell_keys_path, 'rm-key', key_id] + args << key_content if key_content + + gitlab_shell_fast_execute(args) end # Remove all ssh keys from gitlab shell @@ -185,7 +186,7 @@ module Gitlab # remove_all_keys # def remove_all_keys - Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear']) + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end # Add empty directory for storing repositories @@ -267,5 +268,31 @@ module Gitlab def gitlab_shell_keys_path File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + + private + + def gitlab_shell_fast_execute(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + return true if status.zero? + + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + false + end + + def gitlab_shell_fast_execute_raise_error(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + raise Error, output unless status.zero? + true + end + + def gitlab_shell_fast_execute_helper(cmd) + vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS) + + # Don't pass along the entire parent environment to prevent gitlab-shell + # from wasting I/O by searching through GEM_PATH + Bundler.with_original_env { Popen.popen(cmd, nil, vars) } + end end end diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 27696436574..e13808a2720 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -2,7 +2,7 @@ module Gitlab module SlashCommands module Presenters class Base - include Gitlab::Routing.url_helpers + include Gitlab::Routing def initialize(resource = nil) @resource = resource diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb new file mode 100644 index 00000000000..5e89e12b2b1 --- /dev/null +++ b/lib/gitlab/sql/glob.rb @@ -0,0 +1,22 @@ +module Gitlab + module SQL + module Glob + extend self + + # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern + # with SQL expression + def to_like(pattern) + <<~SQL + REPLACE(REPLACE(REPLACE(#{pattern}, + #{q('%')}, #{q('\\%')}), + #{q('_')}, #{q('\\_')}), + #{q('*')}, #{q('%')}) + SQL + end + + def q(string) + ActiveRecord::Base.connection.quote(string) + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 23af9318d1a..35792d2d67f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Gitlab::Routing.url_helpers + include Gitlab::Routing include GitlabRoutingHelper include ActionView::RecordIdentifier @@ -23,9 +23,9 @@ module Gitlab when WikiPage wiki_page_url when ProjectSnippet - project_snippet_url(object) + project_snippet_url(object.project, object) when Snippet - personal_snippet_url(object) + snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end @@ -65,13 +65,13 @@ module Gitlab if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) else - project_snippet_url(snippet, anchor: dom_id(object)) + project_snippet_url(snippet.project, snippet, anchor: dom_id(object)) end end end def wiki_page_url - namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + project_wiki_url(object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f96ee69096d..4aef23b6aee 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -25,27 +25,25 @@ module Gitlab RepoPath: repo_path } - if Gitlab.config.gitaly.enabled - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - - feature_enabled = case action.to_s - when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) - when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) - when 'info_refs' - true - else - raise "Unsupported action: #{action}" - end - if feature_enabled - params[:GitalyAddress] = server[:address] # This field will be deprecated - params[:GitalyServer] = server - end + server = { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + if feature_enabled + params[:GitalyAddress] = server[:address] # This field will be deprecated + params[:GitalyServer] = server end params diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb index 99f9c2c9b04..7cfe76b7b71 100644 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ b/lib/peek/rblineprof/custom_controller_helpers.rb @@ -41,9 +41,14 @@ module Peek ] end.sort_by{ |a,b,c,d,e,f| -f } - output = '' - per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| + output = "<div class='modal-dialog modal-full'><div class='modal-content'>" + output << "<div class='modal-header'>" + output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>" + output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" + output << "</div>" + output << "<div class='modal-body'>" + per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| output << "<div class='peek-rblineprof-file'><div class='heading'>" show_src = file_sort > min @@ -86,11 +91,32 @@ module Peek output << "</div></div>" # .data then .peek-rblineprof-file end - response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe + output << "</div></div></div>" + + response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe end ret end + + private + + def human_description(lineprofiler_param) + case lineprofiler_param + when 'app' + 'app/ & lib/' + when 'views' + 'app/view/' + when 'gems' + 'vendor/gems' + when 'all' + 'everything in Rails.root' + when 'stdlib' + 'everything in the Ruby standard library' + else + 'app/, config/, lib/, vendor/ & plugin/' + end + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e3883278886..e9fb6a008b0 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -42,8 +42,7 @@ namespace :gitlab do http_clone_url = project.http_url_to_repo ssh_clone_url = project.ssh_url_to_repo - omniauth_providers = Gitlab.config.omniauth.providers - omniauth_providers.map! { |provider| provider['name'] } + omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] } puts "" puts "GitLab information".color(:yellow) |