diff options
Diffstat (limited to 'lib')
86 files changed, 1710 insertions, 786 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index efcf0976a81..982a2b88d62 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,7 @@ module API include APIGuard allow_access_with_scope :api + prefix :api version %w(v3 v4), using: :path @@ -47,8 +48,8 @@ module API end before { header['X-Frame-Options'] = 'SAMEORIGIN' } - before { Gitlab::I18n.locale = current_user&.preferred_language } + # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } rescue_from Gitlab::Access::AccessDeniedError do @@ -85,6 +86,9 @@ module API helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers + NO_SLASH_URL_PART_REGEX = %r{[^/]+} + PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji @@ -109,7 +113,8 @@ module API mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::Milestones + mount ::API::ProjectMilestones + mount ::API::GroupMilestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -118,6 +123,7 @@ module API mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets + mount ::API::ProtectedBranches mount ::API::Repositories mount ::API::Runner mount ::API::Runners @@ -134,6 +140,7 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::GroupVariables mount ::API::Version route :any, '*path' do diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 3d816f8771d..d3dbf941298 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -4,19 +4,21 @@ module API class Branches < Grape::API include PaginationParams + BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository branches' do success Entities::RepoBranch end params do use :pagination end - get ":id/repository/branches" do + get ':id/repository/branches' do branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) present paginate(branches), with: Entities::RepoBranch, project: user_project @@ -28,13 +30,14 @@ module API params do requires :branch, type: String, desc: 'The name of the branch' end - get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do + get ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do branch = user_project.repository.find_branch(params[:branch]) not_found!("Branch") unless branch present branch, with: Entities::RepoBranch, project: user_project end + # Note: This API will be deprecated in favor of the protected branches API. # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. @@ -46,7 +49,7 @@ module API optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end - put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do + put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -63,9 +66,9 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = if protected_branch - ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + ::ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) else - ProtectedBranches::ApiCreateService.new(*service_args).execute + ::ProtectedBranches::ApiCreateService.new(*service_args).execute end if protected_branch.valid? @@ -75,13 +78,14 @@ module API end end + # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do success Entities::RepoBranch end params do requires :branch, type: String, desc: 'The name of the branch' end - put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do + put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -99,7 +103,7 @@ module API requires :branch, type: String, desc: 'The name of the branch' requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' end - post ":id/repository/branches" do + post ':id/repository/branches' do authorize_push_project result = CreateBranchService.new(user_project, current_user) @@ -118,7 +122,7 @@ module API params do requires :branch, type: String, desc: 'The name of the branch' end - delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do + delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project result = DeleteBranchService.new(user_project, current_user) @@ -130,7 +134,7 @@ module API end desc 'Delete all merged branches' - delete ":id/repository/merged_branches" do + delete ':id/repository/merged_branches' do DeleteMergedBranchesService.new(user_project, current_user).async_execute accepted! diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1719e9f7205..298831a8fdb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -82,6 +82,38 @@ module API end class Project < Grape::Entity + include ::API::Helpers::RelatedResourcesHelpers + + expose :_links do + expose :self do |project| + expose_url(api_v4_projects_path(id: project.id)) + end + + expose :issues, if: -> (*args) { issues_available?(*args) } do |project| + expose_url(api_v4_projects_issues_path(id: project.id)) + end + + expose :merge_requests, if: -> (*args) { mrs_available?(*args) } do |project| + expose_url(api_v4_projects_merge_requests_path(id: project.id)) + end + + expose :repo_branches do |project| + expose_url(api_v4_projects_repository_branches_path(id: project.id)) + end + + expose :labels do |project| + expose_url(api_v4_projects_labels_path(id: project.id)) + end + + expose :events do |project| + expose_url(api_v4_projects_events_path(id: project.id)) + end + + expose :members do |project| + expose_url(api_v4_projects_members_path(id: project.id)) + end + end + expose :id, :description, :default_branch, :tag_list expose :archived?, as: :archived expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url @@ -208,7 +240,7 @@ module API end expose :protected do |repo_branch, options| - ProtectedBranch.protected?(options[:project], repo_branch.name) + ::ProtectedBranch.protected?(options[:project], repo_branch.name) end expose :developers_can_push do |repo_branch, options| @@ -267,10 +299,23 @@ module API expose :deleted_file?, as: :deleted_file end + class ProtectedRefAccess < Grape::Entity + expose :access_level + expose :access_level_description do |protected_ref_access| + protected_ref_access.humanize + end + end + + class ProtectedBranch < Grape::Entity + expose :name + expose :push_access_levels, using: Entities::ProtectedRefAccess + expose :merge_access_levels, using: Entities::ProtectedRefAccess + end + class Milestone < Grape::Entity expose :id, :iid - expose(:project_id) { |entity| entity&.project_id } - expose(:group_id) { |entity| entity&.group_id } + expose :project_id, if: -> (entity, options) { entity&.project_id } + expose :group_id, if: -> (entity, options) { entity&.group_id } expose :title, :description expose :state, :created_at, :updated_at expose :due_date @@ -297,6 +342,26 @@ module API end class Issue < IssueBasic + include ::API::Helpers::RelatedResourcesHelpers + + expose :_links do + expose :self do |issue| + expose_url(api_v4_project_issue_path(id: issue.project_id, issue_iid: issue.iid)) + end + + expose :notes do |issue| + expose_url(api_v4_projects_issues_notes_path(id: issue.project_id, noteable_id: issue.iid)) + end + + expose :award_emoji do |issue| + expose_url(api_v4_projects_issues_award_emoji_path(id: issue.project_id, issue_iid: issue.iid)) + end + + expose :project do |issue| + expose_url(api_v4_projects_path(id: issue.project_id)) + end + end + expose :subscribed do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end @@ -431,7 +496,7 @@ module API class Event < Grape::Entity expose :title, :project_id, :action_name - expose :target_id, :target_type, :author_id + expose :target_id, :target_iid, :target_type, :author_id expose :data, :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } @@ -619,43 +684,14 @@ module API class ApplicationSetting < Grape::Entity expose :id - expose :default_projects_limit - expose :signup_enabled - expose :password_authentication_enabled - expose :password_authentication_enabled, as: :signin_enabled - expose :gravatar_enabled - expose :sign_in_text - expose :after_sign_up_text - expose :created_at - expose :updated_at - expose :home_page_url - expose :default_branch_protection + expose(*::ApplicationSettingsHelper.visible_attributes) expose(:restricted_visibility_levels) do |setting, _options| setting.restricted_visibility_levels.map { |level| Gitlab::VisibilityLevel.string_level(level) } end - expose :max_attachment_size - expose :session_expire_delay expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } - expose :default_artifacts_expire_in - expose :domain_whitelist - expose :domain_blacklist_enabled - expose :domain_blacklist - expose :user_oauth_applications - expose :after_sign_out_path - expose :container_registry_token_expire_delay - expose :repository_storage - expose :repository_storages - expose :koding_enabled - expose :koding_url - expose :plantuml_enabled - expose :plantuml_url - expose :terminal_max_session_time - expose :polling_interval_multiplier - expose :help_page_hide_commercial_content - expose :help_page_text - expose :help_page_support_url + expose :password_authentication_enabled, as: :signin_enabled end class Release < Grape::Entity diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb new file mode 100644 index 00000000000..b85eb59dc0a --- /dev/null +++ b/lib/api/group_milestones.rb @@ -0,0 +1,85 @@ +module API + class GroupMilestones < Grape::API + include MilestoneResponses + include PaginationParams + + before do + authenticate! + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Get a list of group milestones' do + success Entities::Milestone + end + params do + use :list_params + end + get ":id/milestones" do + list_milestones_for(user_group) + end + + desc 'Get a single group milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + end + get ":id/milestones/:milestone_id" do + authorize! :read_group, user_group + + get_milestone_for(user_group) + end + + desc 'Create a new group milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end + post ":id/milestones" do + authorize! :admin_milestones, user_group + + create_milestone_for(user_group) + end + + desc 'Update an existing group milestone' do + success Entities::Milestone + end + params do + use :update_params + end + put ":id/milestones/:milestone_id" do + authorize! :admin_milestones, user_group + + update_milestone_for(user_group) + end + + desc 'Get all issues for a single group milestone' do + success Entities::IssueBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + use :pagination + end + get ":id/milestones/:milestone_id/issues" do + milestone_issuables_for(user_group, :issue) + end + + desc 'Get all merge requests for a single group milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequestBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + milestone_issuables_for(user_group, :merge_request) + end + end + end +end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb new file mode 100644 index 00000000000..f64da4ab77b --- /dev/null +++ b/lib/api/group_variables.rb @@ -0,0 +1,96 @@ +module API + class GroupVariables < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize! :admin_build, user_group } + + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Get group-level variables' do + success Entities::Variable + end + params do + use :pagination + end + get ':id/variables' do + variables = user_group.variables + present paginate(variables), with: Entities::Variable + end + + desc 'Get a specific variable from a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + get ':id/variables/:key' do + key = params[:key] + variable = user_group.variables.find_by(key: key) + + return not_found!('GroupVariable') unless variable + + present variable, with: Entities::Variable + end + + desc 'Create a new variable in a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' + end + post ':id/variables' do + variable_params = declared_params(include_missing: false) + + variable = user_group.variables.create(variable_params) + + if variable.valid? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Update an existing variable from a group' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' + end + put ':id/variables/:key' do + variable = user_group.variables.find_by(key: params[:key]) + + return not_found!('GroupVariable') unless variable + + variable_params = declared_params(include_missing: false).except(:key) + + if variable.update(variable_params) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Delete an existing variable from a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/variables/:key' do + variable = user_group.variables.find_by(key: params[:key]) + not_found!('GroupVariable') unless variable + + status 204 + variable.destroy + end + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 0f4791841d2..99b8b62691f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -16,6 +16,8 @@ module API @current_user = initial_current_user + Gitlab::I18n.locale = @current_user&.preferred_language + sudo! @current_user @@ -25,6 +27,10 @@ module API initial_current_user != current_user end + def user_group + @group ||= find_group!(params[:id]) + end + def user_project @project ||= find_project!(params[:id]) end @@ -332,12 +338,14 @@ module API env['warden'] end + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(env) + end + # Check the Rails session for valid authentication details - # - # Until CSRF protection is added to the API, disallow this method for - # state-changing endpoints def find_user_from_warden - warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) + warden.try(:authenticate) if verified_request? end def initial_current_user diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb new file mode 100644 index 00000000000..1f677529b07 --- /dev/null +++ b/lib/api/helpers/related_resources_helpers.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module RelatedResourcesHelpers + include GrapeRouteHelpers::NamedRouteMatcher + + def issues_available?(project, options) + available?(:issues, project, options[:current_user]) + end + + def mrs_available?(project, options) + available?(:merge_requests, project, options[:current_user]) + end + + def expose_url(path) + url_options = Gitlab::Application.routes.default_url_options + protocol, host, port = url_options.slice(:protocol, :host, :port).values + + URI::HTTP.build(scheme: protocol, host: host, port: port, path: path).to_s + end + + private + + def available?(feature, project, current_user) + project.feature_available?(feature, current_user) + end + end + end +end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 14b26f28ebf..4cec1145f3a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -29,6 +29,10 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' + optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], + desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' use :pagination end @@ -55,9 +59,11 @@ module API optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' use :issues_params + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', + desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do - issues = find_issues(scope: 'authored') + issues = find_issues present paginate(issues), with: Entities::IssueBasic, current_user: current_user end @@ -112,7 +118,7 @@ module API params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - get ":id/issues/:issue_iid" do + get ":id/issues/:issue_iid", as: :api_v4_project_issue do issue = find_project_issue(params[:issue_iid]) present issue, with: Entities::Issue, current_user: current_user, project: user_project end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index f64ac659413..8810d4e441d 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -4,14 +4,77 @@ module API before { authenticate! } + helpers ::Gitlab::IssuableMetadata + + helpers do + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute + .reorder(args[:order_by] => args[:sort]) + merge_requests = paginate(merge_requests) + .preload(:target_project) + + return merge_requests if args[:view] == 'simple' + + merge_requests + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) + end + + params :merge_requests_params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' + optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' + optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' + optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], + desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + use :pagination + end + end + + resource :merge_requests do + desc 'List merge requests' do + success Entities::MergeRequestBasic + end + params do + use :merge_requests_params + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', + desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + end + get do + merge_requests = find_merge_requests + + options = { with: Entities::MergeRequestBasic, + current_user: current_user } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints - helpers ::Gitlab::IssuableMetadata - helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -29,23 +92,6 @@ module API render_api_error!(errors, 400) end - def find_merge_requests(args = {}) - args = params.merge(args) - - args[:milestone_title] = args.delete(:milestone) - args[:label_name] = args.delete(:labels) - - merge_requests = MergeRequestsFinder.new(current_user, args).execute - .reorder(args[:order_by] => args[:sort]) - merge_requests = paginate(merge_requests) - .preload(:target_project) - - return merge_requests if args[:view] == 'simple' - - merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) - end - params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -63,19 +109,8 @@ module API success Entities::MergeRequestBasic end params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return merge requests sorted in `asc` or `desc` order.' + use :merge_requests_params optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' - optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' - optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' - optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' - optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' - use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb new file mode 100644 index 00000000000..ef09d9505d2 --- /dev/null +++ b/lib/api/milestone_responses.rb @@ -0,0 +1,98 @@ +module API + module MilestoneResponses + extend ActiveSupport::Concern + + included do + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + end + + params :list_params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' + optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' + use :pagination + end + + params :update_params do + requires :milestone_id, type: Integer, desc: 'The milestone ID number' + optional :title, type: String, desc: 'The title of the milestone' + optional :state_event, type: String, values: %w[close activate], + desc: 'The state event of the milestone ' + use :optional_params + at_least_one_of :title, :description, :due_date, :state_event + end + + def list_milestones_for(parent) + milestones = parent.milestones + milestones = Milestone.filter_by_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? + milestones = filter_by_search(milestones, params[:search]) if params[:search] + + present paginate(milestones), with: Entities::Milestone + end + + def get_milestone_for(parent) + milestone = parent.milestones.find(params[:milestone_id]) + present milestone, with: Entities::Milestone + end + + def create_milestone_for(parent) + milestone = ::Milestones::CreateService.new(parent, current_user, declared_params).execute + + if milestone.valid? + present milestone, with: Entities::Milestone + else + render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) + end + end + + def update_milestone_for(parent) + milestone = parent.milestones.find(params.delete(:milestone_id)) + + milestone_params = declared_params(include_missing: false) + milestone = ::Milestones::UpdateService.new(parent, current_user, milestone_params).execute(milestone) + + if milestone.valid? + present milestone, with: Entities::Milestone + else + render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) + end + end + + def milestone_issuables_for(parent, type) + milestone = parent.milestones.find(params[:milestone_id]) + + finder_klass, entity = get_finder_and_entity(type) + + params = build_finder_params(milestone, parent) + + issuables = finder_klass.new(current_user, params).execute + present paginate(issuables), with: entity, current_user: current_user + end + + def build_finder_params(milestone, parent) + finder_params = { milestone_title: milestone.title, sort: 'label_priority' } + + if parent.is_a?(Group) + finder_params.merge(group_id: parent.id) + else + finder_params.merge(project_id: parent.id) + end + end + + def get_finder_and_entity(type) + if type == :issue + [IssuesFinder, Entities::IssueBasic] + else + [MergeRequestsFinder, Entities::MergeRequestBasic] + end + end + end + end + end +end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb deleted file mode 100644 index 3541d3c95fb..00000000000 --- a/lib/api/milestones.rb +++ /dev/null @@ -1,154 +0,0 @@ -module API - class Milestones < Grape::API - include PaginationParams - - before { authenticate! } - - helpers do - def filter_milestones_state(milestones, state) - case state - when 'active' then milestones.active - when 'closed' then milestones.closed - else milestones - end - end - - params :optional_params do - optional :description, type: String, desc: 'The description of the milestone' - optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' - optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a list of project milestones' do - success Entities::Milestone - end - params do - optional :state, type: String, values: %w[active closed all], default: 'all', - desc: 'Return "active", "closed", or "all" milestones' - optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' - optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' - use :pagination - end - get ":id/milestones" do - authorize! :read_milestone, user_project - - milestones = user_project.milestones - milestones = filter_milestones_state(milestones, params[:state]) - milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? - milestones = filter_by_search(milestones, params[:search]) if params[:search] - - present paginate(milestones), with: Entities::Milestone - end - - desc 'Get a single project milestone' do - success Entities::Milestone - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - end - get ":id/milestones/:milestone_id" do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - present milestone, with: Entities::Milestone - end - - desc 'Create a new project milestone' do - success Entities::Milestone - end - params do - requires :title, type: String, desc: 'The title of the milestone' - use :optional_params - end - post ":id/milestones" do - authorize! :admin_milestone, user_project - - milestone = ::Milestones::CreateService.new(user_project, current_user, declared_params).execute - - if milestone.valid? - present milestone, with: Entities::Milestone - else - render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) - end - end - - desc 'Update an existing project milestone' do - success Entities::Milestone - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - optional :title, type: String, desc: 'The title of the milestone' - optional :state_event, type: String, values: %w[close activate], - desc: 'The state event of the milestone ' - use :optional_params - at_least_one_of :title, :description, :due_date, :state_event - end - put ":id/milestones/:milestone_id" do - authorize! :admin_milestone, user_project - milestone = user_project.milestones.find(params.delete(:milestone_id)) - - milestone_params = declared_params(include_missing: false) - milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone) - - if milestone.valid? - present milestone, with: Entities::Milestone - else - render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) - end - end - - desc 'Get all issues for a single project milestone' do - success Entities::IssueBasic - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - use :pagination - end - get ":id/milestones/:milestone_id/issues" do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - - finder_params = { - project_id: user_project.id, - milestone_title: milestone.title, - sort: 'label_priority' - } - - issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project - end - - desc 'Get all merge requests for a single project milestone' do - detail 'This feature was introduced in GitLab 9.' - success Entities::MergeRequestBasic - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - use :pagination - end - get ':id/milestones/:milestone_id/merge_requests' do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - - finder_params = { - project_id: user_project.id, - milestone_title: milestone.title, - sort: 'label_priority' - } - - merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute - present paginate(merge_requests), - with: Entities::MergeRequestBasic, - current_user: current_user, - project: user_project - end - end - end -end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb new file mode 100644 index 00000000000..451998c726a --- /dev/null +++ b/lib/api/project_milestones.rb @@ -0,0 +1,91 @@ +module API + class ProjectMilestones < Grape::API + include PaginationParams + include MilestoneResponses + + before do + authenticate! + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a list of project milestones' do + success Entities::Milestone + end + params do + use :list_params + end + get ":id/milestones" do + authorize! :read_milestone, user_project + + list_milestones_for(user_project) + end + + desc 'Get a single project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end + get ":id/milestones/:milestone_id" do + authorize! :read_milestone, user_project + + get_milestone_for(user_project) + end + + desc 'Create a new project milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end + post ":id/milestones" do + authorize! :admin_milestone, user_project + + create_milestone_for(user_project) + end + + desc 'Update an existing project milestone' do + success Entities::Milestone + end + params do + use :update_params + end + put ":id/milestones/:milestone_id" do + authorize! :admin_milestone, user_project + + update_milestone_for(user_project) + end + + desc 'Get all issues for a single project milestone' do + success Entities::IssueBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ":id/milestones/:milestone_id/issues" do + authorize! :read_milestone, user_project + + milestone_issuables_for(user_project, :issue) + end + + desc 'Get all merge requests for a single project milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequestBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + authorize! :read_milestone, user_project + + milestone_issuables_for(user_project, :merge_request) + end + end + end +end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb new file mode 100644 index 00000000000..d742f2e18d0 --- /dev/null +++ b/lib/api/protected_branches.rb @@ -0,0 +1,85 @@ +module API + class ProtectedBranches < Grape::API + include PaginationParams + + BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_admin_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a project's protected branches" do + success Entities::ProtectedBranch + end + params do + use :pagination + end + get ':id/protected_branches' do + protected_branches = user_project.protected_branches.preload(:push_access_levels, :merge_access_levels) + + present paginate(protected_branches), with: Entities::ProtectedBranch, project: user_project + end + + desc 'Get a single protected branch' do + success Entities::ProtectedBranch + end + params do + requires :name, type: String, desc: 'The name of the branch or wildcard' + end + get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + protected_branch = user_project.protected_branches.find_by!(name: params[:name]) + + present protected_branch, with: Entities::ProtectedBranch, project: user_project + end + + desc 'Protect a single branch or wildcard' do + success Entities::ProtectedBranch + end + params do + requires :name, type: String, desc: 'The name of the protected branch' + optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, + values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + desc: 'Access levels allowed to push (defaults: `40`, master access level)' + optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, + values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + desc: 'Access levels allowed to merge (defaults: `40`, master access level)' + end + post ':id/protected_branches' do + protected_branch = user_project.protected_branches.find_by(name: params[:name]) + if protected_branch + conflict!("Protected branch '#{params[:name]}' already exists") + end + + protected_branch_params = { + name: params[:name], + push_access_levels_attributes: [{ access_level: params[:push_access_level] }], + merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] + } + + service_args = [user_project, current_user, protected_branch_params] + + protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + + if protected_branch.persisted? + present protected_branch, with: Entities::ProtectedBranch, project: user_project + else + render_api_error!(protected_branch.errors.full_messages, 422) + end + end + + desc 'Unprotect a single branch' + params do + requires :name, type: String, desc: 'The name of the protected branch' + end + delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + protected_branch = user_project.protected_branches.find_by!(name: params[:name]) + + protected_branch.destroy + + status 204 + end + end + end +end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 405d25ca3c1..88fc62d33df 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -90,7 +90,7 @@ module API if result.valid? if result.build Gitlab::Metrics.add_event(:build_found, - project: result.build.project.path_with_namespace) + project: result.build.project.full_path) present result.build, with: Entities::JobRequest::Response else Gitlab::Metrics.add_event(:build_not_found) @@ -119,7 +119,7 @@ module API job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, - project: job.project.path_with_namespace) + project: job.project.full_path) case params[:state].to_s when 'success' diff --git a/lib/api/services.rb b/lib/api/services.rb index 7488f95a9b7..843c05ae32e 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -313,12 +313,6 @@ module API desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { - required: true, - name: :project_key, - type: String, - desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' - }, - { required: false, name: :username, type: String, diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b19095d1252..d55a61fa638 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -20,59 +20,6 @@ module API success Entities::ApplicationSetting end params do - # CE - at_least_one_of_ce = [ - :admin_notification_email, - :after_sign_out_path, - :after_sign_up_text, - :akismet_enabled, - :container_registry_token_expire_delay, - :default_artifacts_expire_in, - :default_branch_protection, - :default_group_visibility, - :default_project_visibility, - :default_projects_limit, - :default_snippet_visibility, - :disabled_oauth_sign_in_sources, - :domain_blacklist_enabled, - :domain_whitelist, - :email_author_in_body, - :enabled_git_access_protocol, - :gravatar_enabled, - :help_page_hide_commercial_content, - :help_page_text, - :help_page_support_url, - :home_page_url, - :housekeeping_enabled, - :html_emails_enabled, - :import_sources, - :koding_enabled, - :max_artifacts_size, - :max_attachment_size, - :max_pages_size, - :metrics_enabled, - :plantuml_enabled, - :polling_interval_multiplier, - :recaptcha_enabled, - :repository_checks_enabled, - :repository_storage, - :require_two_factor_authentication, - :restricted_visibility_levels, - :send_user_confirmation_email, - :sentry_enabled, - :clientside_sentry_enabled, - :session_expire_delay, - :shared_runners_enabled, - :sidekiq_throttling_enabled, - :sign_in_text, - :password_authentication_enabled, - :signin_enabled, - :signup_enabled, - :terminal_max_session_time, - :user_default_external, - :user_oauth_applications, - :version_check_enabled - ] optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' @@ -151,7 +98,7 @@ module API given clientside_sentry_enabled: ->(val) { val } do requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' end - optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :koding_enabled, type: Boolean, desc: 'Enable Koding' given koding_enabled: ->(val) { val } do @@ -174,7 +121,8 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' - at_least_one_of(*at_least_one_of_ce) + optional(*::ApplicationSettingsHelper.visible_attributes) + at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) end put "application/settings" do attrs = declared_params(include_missing: false) diff --git a/lib/api/todos.rb b/lib/api/todos.rb index d1f7e364029..55191169dd4 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -59,10 +59,10 @@ module API requires :id, type: Integer, desc: 'The ID of the todo being marked as done' end post ':id/mark_as_done' do - todo = current_user.todos.find(params[:id]) - TodoService.new.mark_todos_as_done([todo], current_user) + TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) + todo = Todo.find(params[:id]) - present todo.reload, with: Entities::Todo, current_user: current_user + present todo, with: Entities::Todo, current_user: current_user end desc 'Mark all todos as done' diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index cb0619105e1..edfdb63d183 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,24 +15,22 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do - project = find_project(params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) - if trigger_request - present trigger_request.pipeline, with: Entities::Pipeline + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute + not_found! unless result + + if result[:http_status] + render_api_error!(result[:message], result[:http_status]) else - errors = 'No pipeline created' - render_api_error!(errors, 400) + present result[:pipeline], with: Entities::Pipeline end end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 3759250f7f6..773f667abe0 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -259,11 +259,40 @@ module API expose :job_events, as: :build_events end - class Issue < ::API::Entities::Issue + class ProjectEntity < Grape::Entity + expose :id, :iid + expose(:project_id) { |entity| entity&.project.try(:id) } + expose :title, :description + expose :state, :created_at, :updated_at + end + + class IssueBasic < ProjectEntity + expose :label_names, as: :labels + expose :milestone, using: ::API::Entities::Milestone + expose :assignees, :author, using: ::API::Entities::UserBasic + + expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + issue.assignees.first + end + + expose :user_notes_count + expose :upvotes, :downvotes + expose :due_date + expose :confidential + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end + end + + class Issue < IssueBasic unexpose :assignees expose :assignee do |issue, options| ::API::Entities::UserBasic.represent(issue.assignees.first, options) end + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user], options[:project] || issue.project) + end end end end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb index 94614bfc8b6..51014591a93 100644 --- a/lib/api/v3/project_hooks.rb +++ b/lib/api/v3/project_hooks.rb @@ -56,7 +56,9 @@ module API use :project_hook_properties end post ":id/hooks" do - hook = user_project.hooks.new(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) + hook = user_project.hooks.new(attrs) if hook.save present hook, with: ::API::V3::Entities::ProjectHook @@ -77,7 +79,9 @@ module API put ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - if hook.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) + if hook.update_attributes(attrs) present hook, with: ::API::V3::Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb index e3b311d61cd..2f2cf259987 100644 --- a/lib/api/v3/todos.rb +++ b/lib/api/v3/todos.rb @@ -11,10 +11,10 @@ module API requires :id, type: Integer, desc: 'The ID of the todo being marked as done' end delete ':id' do - todo = current_user.todos.find(params[:id]) - TodoService.new.mark_todos_as_done([todo], current_user) + TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) + todo = Todo.find(params[:id]) - present todo.reload, with: ::API::Entities::Todo, current_user: current_user + present todo, with: ::API::Entities::Todo, current_user: current_user end desc 'Mark all todos as done' diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index a23d6b6b48c..e9d4c35307b 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -28,12 +28,13 @@ module API end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) - if trigger_request - present trigger_request, with: ::API::V3::Entities::TriggerRequest + result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref].to_s, variables) + pipeline = result.pipeline + + if pipeline.persisted? + present result.trigger_request, with: ::API::V3::Entities::TriggerRequest else - errors = 'No builds created' - render_api_error!(errors, 400) + render_validation_error!(pipeline) end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f755c99ea4a..ca6d6848d41 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -8,18 +8,9 @@ module Backup # Make sure there is a connection ActiveRecord::Base.connection.reconnect! - # saving additional informations - s = {} - s[:db_version] = "#{ActiveRecord::Migrator.current_version}" - s[:backup_created_at] = Time.now - s[:gitlab_version] = Gitlab::VERSION - s[:tar_version] = tar_version - s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(backup_path) do File.open("#{backup_path}/backup_information.yml", "w+") do |file| - file << s.to_yaml.gsub(/^---\n/, '') + file << backup_information.to_yaml.gsub(/^---\n/, '') end # create archive @@ -33,11 +24,11 @@ module Backup abort 'Backup failed' end - upload(tar_file) + upload end end - def upload(tar_file) + def upload $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection @@ -48,7 +39,7 @@ module Backup directory = connect_to_remote_directory(connection_settings) - if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, + if directory.files.create(key: remote_target, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, storage_class: Gitlab.config.backup.upload.storage_class) @@ -177,7 +168,8 @@ module Backup end def connect_to_remote_directory(connection_settings) - connection = ::Fog::Storage.new(connection_settings) + # our settings use string keys, but Fog expects symbols + connection = ::Fog::Storage.new(connection_settings.symbolize_keys) # We only attempt to create the directory for local backups. For AWS # and other cloud providers, we cannot guarantee the user will have @@ -193,6 +185,14 @@ module Backup Gitlab.config.backup.upload.remote_directory end + def remote_target + if ENV['DIRECTORY'] + File.join(ENV['DIRECTORY'], tar_file) + else + tar_file + end + end + def backup_contents folders_to_backup + archives_to_backup + ["backup_information.yml"] end @@ -214,5 +214,19 @@ module Backup def settings @settings ||= YAML.load_file("backup_information.yml") end + + def tar_file + @tar_file ||= "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}" + end + + def backup_information + @backup_information ||= { + db_version: ActiveRecord::Migrator.current_version.to_s, + backup_created_at: Time.now, + gitlab_version: Gitlab::VERSION, + tar_version: tar_version, + skipped: ENV["SKIP"] + } + end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index a1685c77916..88821ae56e0 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,7 +7,7 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.full_path} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -42,7 +42,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.full_path} ... " if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else @@ -71,11 +71,11 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.full_path} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_dir_exist + project.ensure_storage_path_exist cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) @@ -104,7 +104,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.full_path} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -142,11 +142,11 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + '.bundle') + File.join(backup_repos_path, project.disk_path + '.bundle') end def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.path_with_namespace) + path = File.join(backup_repos_path, project.disk_path) if dir File.join(path, "#{dir}.tar") @@ -185,13 +185,14 @@ module Backup def progress_warn(project, cmd, output) progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + progress.puts "Ignoring error on #{project.full_path} - #{output}".color(:orange) end def empty_repo?(project_or_wiki) + project_or_wiki.repository.expire_exists_cache # protect backups from stale cache project_or_wiki.repository.empty_repo? rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange) + progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.full_path} - #{e.message}".color(:orange) false end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 7a262dd025c..685b43605ae 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -259,7 +259,7 @@ module Banzai found = [] projects.each do |project| - ref = project.path_with_namespace + ref = project.full_path get_or_set_cache(cache, ref) { project } found << ref end @@ -277,7 +277,7 @@ module Banzai end def current_project_path - @current_project_path ||= project.path_with_namespace + @current_project_path ||= project.full_path end def current_project_namespace_path diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index c2fed57a0d8..758f15c8a67 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -51,7 +51,7 @@ module Banzai uri.path = [ relative_url_root, - context[:project].path_with_namespace, + context[:project].full_path, uri_type(file_path), Addressable::URI.escape(ref), Addressable::URI.escape(file_path) diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 45bb66dc99f..09844931be5 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -28,7 +28,7 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) + File.join(Gitlab.config.gitlab.url, project.full_path, uri) end def project diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7801cb5baf..ad08c0905e2 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -132,6 +132,8 @@ module Banzai end def self.cacheless_render(text, context = {}) + return text.to_s unless text.present? + Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index e2e91ce99cd..79058c02ce5 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -29,7 +29,7 @@ module Ci if result.valid? if result.build Gitlab::Metrics.add_event(:build_found, - project: result.build.project.path_with_namespace) + project: result.build.project.full_path) present result.build, with: Entities::BuildDetails else @@ -64,7 +64,7 @@ module Ci build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, - project: build.project.path_with_namespace) + project: build.project.full_path) case params[:state].to_s when 'success' diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb index 6e622601680..6225203f223 100644 --- a/lib/ci/api/triggers.rb +++ b/lib/ci/api/triggers.rb @@ -24,12 +24,13 @@ module Ci end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref], variables) - if trigger_request - present trigger_request, with: Entities::TriggerRequest + result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) + pipeline = result.pipeline + + if pipeline.persisted? + present result.trigger_request, with: Entities::TriggerRequest else - errors = 'No builds created' - render_api_error!(errors, 400) + render_validation_error!(pipeline) end end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index b1eb1a6cef1..ae65653645b 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -28,7 +28,13 @@ module DeclarativePolicy subject = find_delegate(subject) - class_for_class(subject.class) + policy_class = class_for_class(subject.class) + raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class + end + + def has_policy?(subject) + !class_for_class(subject.class).nil? end private @@ -51,9 +57,7 @@ module DeclarativePolicy end end - policy_class = subject_class.instance_variable_get(CLASS_CACHE_IVAR) - raise "no policy for #{subject.class.name}" if policy_class.nil? - policy_class + subject_class.instance_variable_get(CLASS_CACHE_IVAR) end def compute_class_for_class(subject_class) @@ -71,6 +75,8 @@ module DeclarativePolicy nil end end + + nil end def find_delegate(subject) diff --git a/lib/github/import.rb b/lib/github/import.rb index ff5d7db2705..cea4be5460b 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -93,7 +93,7 @@ module Github def fetch_wiki_repository wiki_url = "https://#{options.fetch(:token)}@github.com/#{repo}.wiki.git" - wiki_path = "#{project.path_with_namespace}.wiki" + wiki_path = "#{project.full_path}.wiki" unless project.wiki.repository_exists? gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb new file mode 100644 index 00000000000..0fbc6b70989 --- /dev/null +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -0,0 +1,107 @@ +module Gitlab + module BackgroundMigration + class DeserializeMergeRequestDiffsAndCommits + attr_reader :diff_ids, :commit_rows, :file_rows + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + BUFFER_ROWS = 1000 + + def perform(start_id, stop_id) + merge_request_diffs = MergeRequestDiff + .select(:id, :st_commits, :st_diffs) + .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') + .where(id: start_id..stop_id) + + reset_buffers! + + merge_request_diffs.each do |merge_request_diff| + commits, files = single_diff_rows(merge_request_diff) + + diff_ids << merge_request_diff.id + commit_rows.concat(commits) + file_rows.concat(files) + + if diff_ids.length > BUFFER_ROWS || + commit_rows.length > BUFFER_ROWS || + file_rows.length > BUFFER_ROWS + + flush_buffers! + end + end + + flush_buffers! + end + + private + + def reset_buffers! + @diff_ids = [] + @commit_rows = [] + @file_rows = [] + end + + def flush_buffers! + if diff_ids.any? + MergeRequestDiff.transaction do + Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) + Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + end + end + + reset_buffers! + end + + def single_diff_rows(merge_request_diff) + sha_attribute = Gitlab::Database::ShaAttribute.new + commits = YAML.load(merge_request_diff.st_commits) rescue [] + + commit_rows = commits.map.with_index do |commit, index| + commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) + sha = commit_hash.delete(:id) + + commit_hash.merge( + merge_request_diff_id: merge_request_diff.id, + relative_order: index, + sha: sha_attribute.type_cast_for_database(sha) + ) + end + + diffs = YAML.load(merge_request_diff.st_diffs) rescue [] + diffs = [] unless valid_raw_diffs?(diffs) + + file_rows = diffs.map.with_index do |diff, index| + diff_hash = diff.to_hash.with_indifferent_access.merge( + binary: false, + merge_request_diff_id: merge_request_diff.id, + relative_order: index + ) + + # Compatibility with old diffs created with Psych. + diff_hash.tap do |hash| + diff_text = hash[:diff] + + if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? + hash[:binary] = true + hash[:diff] = [diff_text].pack('m0') + end + end + end + + [commit_rows, file_rows] + end + + # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as + # valid, because we don't render them usefully anyway. + def valid_raw_diffs?(diffs) + return false unless diffs.respond_to?(:each) + + diffs.all? { |diff| diff.is_a?(Hash) } + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 5a6d9ae99a0..28bbf3b384e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,7 +61,7 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.path_with_namespace}.wiki" + path_with_namespace = "#{project.full_path}.wiki" import_url = project.import_url.sub(/\.git\z/, ".git/wiki") gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) rescue StandardError => e diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index bf557103cfd..0735243e021 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -48,7 +48,7 @@ module Gitlab end def starting_month - Date.today.month + Date.current.month end private @@ -66,12 +66,18 @@ module Gitlab .select(:id) conditions = t[:created_at].gteq(date_from.beginning_of_day) - .and(t[:created_at].lteq(Date.today.end_of_day)) + .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) + date_interval = if Gitlab::Database.postgresql? + "INTERVAL '#{Time.zone.now.utc_offset} seconds'" + else + "INTERVAL #{Time.zone.now.utc_offset} SECOND" + end + Event.reorder(nil) - .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount') - .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)') + .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') + .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") .where(conditions) .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 8c8729b6557..5c5f507d44d 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -24,11 +24,11 @@ module Gitlab # total_commits_count: Fixnum # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) commits = Array(commits) # Total commits count - commits_count = commits.size + commits_count ||= commits.size # Get latest 20 commits ASC commits_limited = commits.last(20) diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb deleted file mode 100644 index a78fde9d782..00000000000 --- a/lib/gitlab/devise_failure.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - class DeviseFailure < Devise::FailureApp - protected - - # Override `Devise::FailureApp#request_format` to handle a special case - # - # This tells Devise to handle an unauthenticated `.zip` request as an HTML - # request (i.e., redirect to sign in). - # - # Otherwise, Devise would respond with a 401 Unauthorized with - # `Content-Type: application/zip` and a response body in plaintext, and the - # browser would freak out. - # - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 - def request_format - if request.format == :zip - Mime::Type.lookup_by_extension(:html).ref - else - super - end - end - end -end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2863a4da71..6d7de52cb80 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -79,13 +79,6 @@ module Gitlab @new_content_sha = refs&.head_sha end - def new_content_commit - return @new_content_commit if defined?(@new_content_commit) - - sha = new_content_commit - @new_content_commit = repository.commit(sha) if sha - end - def old_content_sha return if new_file? return @old_content_sha if defined?(@old_content_sha) @@ -94,13 +87,6 @@ module Gitlab @old_content_sha = refs&.base_sha end - def old_content_commit - return @old_content_commit if defined?(@old_content_commit) - - sha = old_content_sha - @old_content_commit = repository.commit(sha) if sha - end - def new_blob return @new_blob if defined?(@new_blob) @@ -123,10 +109,6 @@ module Gitlab new_content_sha || old_content_sha end - def content_commit - new_content_commit || old_content_commit - end - def blob new_blob || old_blob end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 85e6db0a689..72d7d4f84d1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -181,8 +181,6 @@ module Gitlab end def find_merge_base_with_master(branch:) - return if merge_base_found? - # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 31579e94a87..8eea33b9ab5 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -15,7 +15,6 @@ module Gitlab def execute raise SentNotificationNotFoundError unless sent_notification - raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/ validate_permission!(:create_note) diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index dd1d9dcd555..cd9d3a6483f 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -117,7 +117,7 @@ module Gitlab def subject subject_text = '[Git]' - subject_text << "[#{project.path_with_namespace}]" + subject_text << "[#{project.full_path}]" subject_text << "[#{ref_name}]" if @action == :push subject_text << ' ' diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 0d6b08b5d29..c8f4591d060 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -26,6 +26,9 @@ module Gitlab raise EmptyEmailError if @raw.blank? mail = build_mail + + ignore_auto_submitted!(mail) + mail_key = extract_mail_key(mail) handler = Handler.for(mail, mail_key) @@ -87,6 +90,16 @@ module Gitlab break key if key end end + + def ignore_auto_submitted!(mail) + # Mail::Header#[] is case-insensitive + auto_submitted = mail.header['Auto-Submitted']&.value + + # Mail::Field#value would strip leading and trailing whitespace + raise AutoGeneratedEmailError if + # See also https://tools.ietf.org/html/rfc3834 + auto_submitted && auto_submitted != 'no' + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4175746be39..b6449f27034 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -10,7 +10,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode! ref.sub(/\Arefs\/(tags|heads)\//, '') + encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 0deaab01b5b..8dbe25e55f6 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: needs 1 RPC for #load_blame. - module Gitlab module Git class Blame @@ -26,15 +24,29 @@ module Gitlab private - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) - # Read in binary mode to ensure ASCII-8BIT - raw_output = IO.popen(cmd, 'rb') {|io| io.read } + raw_output = @repo.gitaly_migrate(:blame) do |is_enabled| + if is_enabled + load_blame_by_gitaly + else + load_blame_by_shelling_out + end + end + output = encode_utf8(raw_output) process_raw_blame output end + def load_blame_by_gitaly + @repo.gitaly_commit_client.raw_blame(@sha, @path) + end + + def load_blame_by_shelling_out + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + IO.popen(cmd, 'rb') {|io| io.read } + end + def process_raw_blame(output) lines, final = [], [] info, commits = {}, {} diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 09511cc6504..ca7e3a7c4be 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -319,6 +319,15 @@ module Gitlab end end + # Get the gpg signature of this commit. + # + # Ex. + # commit.signature(repo) + # + def signature(repo) + Rugged::Commit.extract_signature(repo.rugged, sha) + end + def stats Gitlab::Git::CommitStats.new(self) end @@ -327,7 +336,7 @@ module Gitlab begin raw_commit.to_mbox(options) rescue Rugged::InvalidError => ex - if ex.message =~ /Commit \w+ is a merge commit/ + if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 63eebadff2e..734aed8fbc1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,6 +45,8 @@ module Gitlab :bare?, to: :rugged + delegate :exists?, to: :gitaly_repository_client + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -80,10 +82,14 @@ module Gitlab end # Returns an Array of Branches - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389 - def branches(sort_by: nil) - branches_filter(sort_by: sort_by) + def branches + gitaly_migrate(:branches) do |is_enabled| + if is_enabled + gitaly_ref_client.branches + else + branches_filter + end + end end def reload_rugged @@ -162,20 +168,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end + gitaly_migrate(:tags) do |is_enabled| + if is_enabled + tags_from_gitaly + else + tags_from_rugged end - - target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) - end.sort_by(&:name) + end end # Returns true if the given tag exists @@ -208,10 +207,6 @@ module Gitlab !empty? end - def repo_exists? - !!rugged - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -305,17 +300,14 @@ module Gitlab raw_log(options).map { |c| Commit.decorate(c) } end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd += %W[--count #{options[:ref]}] - cmd += %W[-- #{options[:path]}] if options[:path].present? - - raw_output = IO.popen(cmd) { |io| io.read } - - raw_output.to_i + gitaly_migrate(:count_commits) do |is_enabled| + if is_enabled + count_commits_by_gitaly(options) + else + count_commits_by_shelling_out(options) + end + end end def sha_from_ref(ref) @@ -358,6 +350,13 @@ module Gitlab rugged.merge_base(from, to) end + # Gitaly note: JV: check gitlab-ee before removing this method. + def rugged_is_ancestor?(ancestor_id, descendant_id) + return false if ancestor_id.nil? || descendant_id.nil? + + merge_base_commit(ancestor_id, descendant_id) == ancestor_id + end + # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) gitaly_commit_client.is_ancestor(from, to) @@ -480,20 +479,6 @@ module Gitlab end end - # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or - # tag name or a commit SHA. Valid +reset_type+ values are: - # - # [:soft] - # the head will be moved to the commit. - # [:mixed] - # will trigger a +:soft+ reset, plus the index will be replaced - # with the content of the commit tree. - # [:hard] - # will trigger a +:mixed+ reset and the working directory will be - # replaced with the content of the index. (Untracked and ignored files - # will be left alone) - delegate :reset, to: :rugged - # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # @@ -518,154 +503,6 @@ module Gitlab # TODO: implement this method end - # Check out the specified ref. Valid options are: - # - # :b - Create a new branch at +start_point+ and set HEAD to the new - # branch. - # - # * These options are passed to the Rugged::Repository#checkout method: - # - # :progress :: - # A callback that will be executed for checkout progress notifications. - # Up to 3 parameters are passed on each execution: - # - # - The path to the last updated file (or +nil+ on the very first - # invocation). - # - The number of completed checkout steps. - # - The number of total checkout steps to be performed. - # - # :notify :: - # A callback that will be executed for each checkout notification - # types specified with +:notify_flags+. Up to 5 parameters are passed - # on each execution: - # - # - An array containing the +:notify_flags+ that caused the callback - # execution. - # - The path of the current file. - # - A hash describing the baseline blob (or +nil+ if it does not - # exist). - # - A hash describing the target blob (or +nil+ if it does not exist). - # - A hash describing the workdir blob (or +nil+ if it does not - # exist). - # - # :strategy :: - # A single symbol or an array of symbols representing the strategies - # to use when performing the checkout. Possible values are: - # - # :none :: - # Perform a dry run (default). - # - # :safe :: - # Allow safe updates that cannot overwrite uncommitted data. - # - # :safe_create :: - # Allow safe updates plus creation of missing files. - # - # :force :: - # Allow all updates to force working directory to look like index. - # - # :allow_conflicts :: - # Allow checkout to make safe updates even if conflicts are found. - # - # :remove_untracked :: - # Remove untracked files not in index (that are not ignored). - # - # :remove_ignored :: - # Remove ignored files not in index. - # - # :update_only :: - # Only update existing files, don't create new ones. - # - # :dont_update_index :: - # Normally checkout updates index entries as it goes; this stops - # that. - # - # :no_refresh :: - # Don't refresh index/config/etc before doing checkout. - # - # :disable_pathspec_match :: - # Treat pathspec as simple list of exact match file paths. - # - # :skip_locked_directories :: - # Ignore directories in use, they will be left empty. - # - # :skip_unmerged :: - # Allow checkout to skip unmerged files (NOT IMPLEMENTED). - # - # :use_ours :: - # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). - # - # :use_theirs :: - # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). - # - # :update_submodules :: - # Recursively checkout submodules with same options (NOT - # IMPLEMENTED). - # - # :update_submodules_if_changed :: - # Recursively checkout submodules if HEAD moved in super repo (NOT - # IMPLEMENTED). - # - # :disable_filters :: - # If +true+, filters like CRLF line conversion will be disabled. - # - # :dir_mode :: - # Mode for newly created directories. Default: +0755+. - # - # :file_mode :: - # Mode for newly created files. Default: +0755+ or +0644+. - # - # :file_open_flags :: - # Mode for opening files. Default: - # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. - # - # :notify_flags :: - # A single symbol or an array of symbols representing the cases in - # which the +:notify+ callback should be invoked. Possible values are: - # - # :none :: - # Do not invoke the +:notify+ callback (default). - # - # :conflict :: - # Invoke the callback for conflicting paths. - # - # :dirty :: - # Invoke the callback for "dirty" files, i.e. those that do not need - # an update but no longer match the baseline. - # - # :updated :: - # Invoke the callback for any file that was changed. - # - # :untracked :: - # Invoke the callback for untracked files. - # - # :ignored :: - # Invoke the callback for ignored files. - # - # :all :: - # Invoke the callback for all these cases. - # - # :paths :: - # A glob string or an array of glob strings specifying which paths - # should be taken into account for the checkout operation. +nil+ will - # match all files. Default: +nil+. - # - # :baseline :: - # A Rugged::Tree that represents the current, expected contents of the - # workdir. Default: +HEAD+. - # - # :target_directory :: - # A path to an alternative workdir directory in which the checkout - # should be performed. - def checkout(ref, options = {}, start_point = "HEAD") - if options[:b] - rugged.branches.create(ref, start_point) - options.delete(:b) - end - default_options = { strategy: [:recreate_missing, :safe] } - rugged.checkout(ref, default_options.merge(options)) - end - # Delete the specified branch from the repository def delete_branch(branch_name) rugged.branches.delete(branch_name) @@ -803,6 +640,33 @@ module Gitlab @attributes.attributes(path) end + def languages(ref = nil) + Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| + if is_enabled + gitaly_commit_client.languages(ref) + else + ref ||= rugged.head.target_id + languages = Linguist::Repository.new(rugged, ref).languages + total = languages.map(&:last).sum + + languages = languages.map do |language| + name, share = language + color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" + { + value: (share.to_f * 100 / total).round(2), + label: name, + color: color, + highlight: color + } + end + + languages.sort do |x, y| + y[:value] <=> x[:value] + end + end + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end @@ -815,6 +679,18 @@ module Gitlab @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) end + def gitaly_repository_client + @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) + end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + private # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. @@ -1113,12 +989,46 @@ module Gitlab end end - def gitaly_migrate(method, &block) - Gitlab::GitalyClient.migrate(method, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) + def tags_from_rugged + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + end.sort_by(&:name) + end + + def last_commit_for_path_by_rugged(sha, path) + sha = last_commit_id_for_path(sha, path) + commit(sha) + end + + def tags_from_gitaly + gitaly_ref_client.tags + end + + def count_commits_by_gitaly(options) + gitaly_commit_client.commit_count(options[:ref], options) + end + + def count_commits_by_shelling_out(options) + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd += %W[--count #{options[:ref]}] + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + + raw_output.to_i end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 435e41e36fb..c90ef282fdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -57,7 +57,7 @@ module Gitlab metadata = yield(metadata) if block_given? stub(service, storage).send(rpc, request, metadata) end - + def self.request_metadata(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) { metadata: { 'authorization' => "Bearer #{encoded_token}" } } diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index c6e52b530b3..ac6817e6d0e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -85,15 +85,32 @@ module Gitlab end end - def commit_count(ref) + def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, revision: ref ) + request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? + request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? + request.path = options[:path] if options[:path].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count end + def last_commit_for_path(revision, path) + request = Gitaly::LastCommitForPathRequest.new( + repository: @gitaly_repo, + revision: revision.force_encoding(Encoding::ASCII_8BIT), + path: path.to_s.force_encoding(Encoding::ASCII_8BIT) + ) + + gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit + return unless gitaly_commit + + commit = GitalyClient::Commit.new(@repository, gitaly_commit) + Gitlab::Git::Commit.new(commit) + end + def between(from, to) request = Gitaly::CommitsBetweenRequest.new( repository: @gitaly_repo, @@ -118,6 +135,24 @@ module Gitlab consume_commits_response(response) end + def languages(ref = nil) + request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') + response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) + + response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } + end + + def raw_blame(revision, path) + request = Gitaly::RawBlameRequest.new( + repository: @gitaly_repo, + revision: revision, + path: path + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) + response.reduce("") { |memo, msg| memo << msg.data } + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 2c3d53410ac..b0f7548b7dc 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -10,6 +10,19 @@ module Gitlab @storage = repository.storage end + def branches + request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + response.flat_map do |message| + message.branches.map do |branch| + gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) + target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) @@ -52,6 +65,12 @@ module Gitlab consume_branches_response(response) end + def tags + request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + consume_tags_response(response) + end + private def consume_refs_response(response) @@ -79,6 +98,25 @@ module Gitlab end end + def consume_tags_response(response) + response.flat_map do |message| + message.tags.map do |gitaly_tag| + if gitaly_tag.target_commit.present? + commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) + gitaly_commit = Gitlab::Git::Commit.new(commit) + end + + Gitlab::Git::Tag.new( + @repository, + encode!(gitaly_tag.name.dup), + gitaly_tag.id, + gitaly_commit, + encode!(gitaly_tag.message.chomp) + ) + end + end + end + def commit_from_local_branches_response(response) # Git messages have no encoding enforcements. However, in the UI we only # handle UTF-8, so basically we cross our fingers that the message force diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb new file mode 100644 index 00000000000..13e75b256a7 --- /dev/null +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -0,0 +1,32 @@ +module Gitlab + module GitalyClient + class RepositoryService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def exists? + request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :repository_service, :exists, request).exists + end + + def garbage_collect(create_bitmap) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request) + end + + def repack_full(create_bitmap) + request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :repack_full, request) + end + + def repack_incremental + request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request) + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index a8c0b47e786..266b1a6fece 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -254,7 +254,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6c592ff469c..0396122eeb9 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -7,8 +7,8 @@ module Gitlab @project = project end - def path_with_namespace - "#{project.path_with_namespace}.wiki" + def disk_path + "#{project.disk_path}.wiki" end def import_url diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb new file mode 100644 index 00000000000..e1d1724295a --- /dev/null +++ b/lib/gitlab/gpg.rb @@ -0,0 +1,62 @@ +module Gitlab + module Gpg + extend self + + module CurrentKeyChain + extend self + + def add(key) + GPGME::Key.import(key) + end + + def fingerprints_from_key(key) + import = GPGME::Key.import(key) + + return [] if import.imported == 0 + + import.imports.map(&:fingerprint) + end + end + + def fingerprints_from_key(key) + using_tmp_keychain do + CurrentKeyChain.fingerprints_from_key(key) + end + end + + def primary_keyids_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).map { |raw_key| raw_key.primary_subkey.keyid } + end + end + + def user_infos_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| + raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + end + end + end + + def using_tmp_keychain + Dir.mktmpdir do |dir| + @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] + @original_dirs.push(dir) + + GPGME::Engine.home_dir = dir + + return_value = yield + + @original_dirs.pop + + GPGME::Engine.home_dir = @original_dirs[-1] + + return_value + end + end + end +end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb new file mode 100644 index 00000000000..55428b85207 --- /dev/null +++ b/lib/gitlab/gpg/commit.rb @@ -0,0 +1,85 @@ +module Gitlab + module Gpg + class Commit + attr_reader :commit + + def initialize(commit) + @commit = commit + + @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + end + + def has_signature? + !!(@signature_text && @signed_text) + end + + def signature + return unless has_signature? + + cached_signature = GpgSignature.find_by(commit_sha: commit.sha) + return cached_signature if cached_signature.present? + + using_keychain do |gpg_key| + create_cached_signature!(gpg_key) + end + end + + def update_signature!(cached_signature) + using_keychain do |gpg_key| + cached_signature.update_attributes!(attributes(gpg_key)) + end + end + + private + + def using_keychain + Gitlab::Gpg.using_tmp_keychain do + # first we need to get the keyid from the signature to query the gpg + # key belonging to the keyid. + # This way we can add the key to the temporary keychain and extract + # the proper signature. + gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + + if gpg_key + Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) + @verified_signature = nil + end + + yield gpg_key + end + end + + def verified_signature + @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + break verified_signature + end + end + + def create_cached_signature!(gpg_key) + GpgSignature.create!(attributes(gpg_key)) + end + + def attributes(gpg_key) + user_infos = user_infos(gpg_key) + + { + commit_sha: commit.sha, + project: commit.project, + gpg_key: gpg_key, + gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_user_name: user_infos[:name], + gpg_key_user_email: user_infos[:email], + valid_signature: gpg_signature_valid_signature_value(gpg_key) + } + end + + def gpg_signature_valid_signature_value(gpg_key) + !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + end + + def user_infos(gpg_key) + gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} + end + end + end +end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb new file mode 100644 index 00000000000..3bb491120ba --- /dev/null +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -0,0 +1,19 @@ +module Gitlab + module Gpg + class InvalidGpgSignatureUpdater + def initialize(gpg_key) + @gpg_key = gpg_key + end + + def run + GpgSignature + .select(:id, :commit_sha, :project_id) + .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) + .find_each do |gpg_signature| + Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7de6d4d9367..8b365dab185 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -27,10 +27,10 @@ module Gitlab Metric.new(name, value, labels) end - def with_timing(proc) + def with_timing start = Time.now - result = proc.call - yield result, Time.now.to_f - start.to_f + result = yield + [result, Time.now.to_f - start.to_f] end def catch_timeout(seconds, &block) diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index bebde857b16..9e91c135956 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,47 +10,45 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - tmp_file_path = tmp_file_path(storage_name) - if !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - elsif !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) else - HealthChecks::Result.new(true, nil, shard: storage_name) + with_temp_file(storage_name) do |tmp_file_path| + if !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + end end rescue RuntimeError => ex message = "unexpected error #{ex} when checking storage #{storage_name}" Rails.logger.error(message) HealthChecks::Result.new(false, message, shard: storage_name) - ensure - delete_test_file(tmp_file_path) end end end def metrics repository_storages.flat_map do |storage_name| - tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + storage_stat_metrics(storage_name), + storage_write_metrics(storage_name), + storage_read_metrics(storage_name) ].flatten end end private - def operation_metrics(ok_metric, latency_metric, operation, **labels) - with_timing operation do |result, elapsed| - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - end + def operation_metrics(ok_metric, latency_metric, **labels) + result, elapsed = yield + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] rescue RuntimeError => ex Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] @@ -68,19 +66,36 @@ module Gitlab Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end - def tmp_file_path(storage_name) - Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + def with_temp_file(storage_name) + temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } + yield temp_file_path + ensure + delete_test_file(temp_file_path) end - def path(storage_name) + def storage_path(storage_name) storages_paths&.dig(storage_name, 'path') end + # All below test methods use shell commands to perform actions on storage volumes. + # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, + # we can rely on shell commands to be terminated once `timeout` kills them. + # + # However we also fallback to pure Ruby file operations in case a specific shell command is missing + # so we are still able to perform healthchecks and gather metrics from such system. + + def delete_test_file(tmp_path) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) + status.zero? + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + def storage_stat_test(storage_name) - stat_path = File.join(path(storage_name), '.') + stat_path = File.join(storage_path(storage_name), '.') begin _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status == 0 + status.zero? rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? end @@ -90,7 +105,7 @@ module Gitlab _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT written_bytes == RANDOM_STRING.length @@ -100,17 +115,33 @@ module Gitlab _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT file_contents = File.read(tmp_path) rescue Errno::ENOENT file_contents == RANDOM_STRING end - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status == 0 - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT + def storage_stat_metrics(storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do + with_timing { storage_stat_test(storage_name) } + end + end + + def storage_write_metrics(storage_name) + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + with_timing { storage_write_test(tmp_file_path) } + end + end + end + + def storage_read_metrics(storage_name) + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + storage_write_test(tmp_file_path) # writes data used by read test + with_timing { storage_read_test(tmp_file_path) } + end + end end end end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 3dcb28a193c..f5026171ba4 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -15,14 +15,13 @@ module Gitlab end def metrics - with_timing method(:check) do |result, elapsed| - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) - [ - metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency_seconds", elapsed) - ] - end + result, elapsed = with_timing(&method(:check)) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency_seconds", elapsed) + ] end private diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index c824d3ea9fc..32ca2809b2f 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle) + gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 62a2553675c..f9ae5079d7c 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,6 +24,7 @@ module Gitlab end def uploads_path + # TODO: decide what to do with uploads. We will use UUIDs here too? File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 7b05290e5cc..8867a91c244 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -101,7 +101,7 @@ module Gitlab end def user_attributes - %W(#{config.uid} cn mail dn) + %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] end end end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index 4745311402c..ed1de73f8c6 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -42,7 +42,7 @@ module Gitlab end def adapter - OmniAuth::LDAP::Adaptor.new(config.options.symbolize_keys) + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) end def config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 6fdf68641e2..c8f19cd52d5 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -2,6 +2,12 @@ module Gitlab module LDAP class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + attr_accessor :provider, :options def self.enabled? @@ -12,6 +18,12 @@ module Gitlab Gitlab.config.ldap.servers.values end + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + def self.providers servers.map { |server| server['provider_name'] } end @@ -39,7 +51,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption + encryption: encryption_options ) opts.merge!(auth_options) if has_auth? @@ -50,9 +62,10 @@ module Gitlab def omniauth_options opts = base_options.merge( base: base, - method: options['method'], + encryption: options['encryption'], filter: omniauth_user_filter, - name_proc: name_proc + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] ) if has_auth? @@ -62,6 +75,9 @@ module Gitlab ) end + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts end @@ -157,15 +173,37 @@ module Gitlab base_config.servers.values.find { |server| server['provider_name'] == provider } end - def encryption - case options['method'].to_s - when 'ssl' - :simple_tls - when 'tls' - :start_tls - else - nil - end + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts end def auth_options diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 3f2bbd9f6a6..e8330917e91 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -166,12 +166,17 @@ module Gitlab username ||= auth_hash.username email ||= auth_hash.email + valid_username = ::Namespace.clean_path(username) + + uniquify = Uniquify.new + valid_username = uniquify.string(valid_username) { |s| !DynamicPathValidator.valid_user_path?(s) } + name = auth_hash.name - name = ::Namespace.clean_path(username) if name.strip.empty? + name = valid_username if name.strip.empty? { name: name, - username: ::Namespace.clean_path(username), + username: valid_username, email: email, password: auth_hash.password, password_confirmation: auth_hash.password, diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 60a32d5d5ea..894bd5efae5 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -14,42 +14,42 @@ module Gitlab TOP_LEVEL_ROUTES = %w[ - .well-known + 404.html + 422.html + 500.html + 502.html + 503.html abuse_reports admin - all api + apple-touch-icon-precomposed.png + apple-touch-icon.png assets autocomplete ci dashboard + deploy.html explore + favicon.ico files groups health_check help - hooks import invites - issues jwt koding - member - merge_requests - new - notes notification_settings oauth profile projects public - repository robots.txt s search sent_notifications - services + slash-command-logo.png snippets - teams u unicorn_test unsubscribes diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 67c69d9ccf3..69d055c901c 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -6,14 +6,13 @@ module Gitlab def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| - query_context = { - environment_slug: deployment.environment.slug, - environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, - timeframe_start: (deployment.created_at - 30.minutes).to_f, - timeframe_end: (deployment.created_at + 30.minutes).to_f - } - - query_metrics(query_context) + query_metrics( + common_query_context( + deployment.environment, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + ) + ) end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index b5a679ddd79..db4708b22e4 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -6,14 +6,9 @@ module Gitlab def query(environment_id) Environment.find_by(id: environment_id).try do |environment| - query_context = { - environment_slug: environment.slug, - environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, - timeframe_start: 8.hours.ago.to_f, - timeframe_end: Time.now.to_f - } - - query_metrics(query_context) + query_metrics( + common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) + ) end end end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index e44be770544..7ac6162b54d 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -42,15 +42,18 @@ module Gitlab end def process_query(context, query) - query_with_result = query.dup + query = query.dup result = if query.key?(:query_range) - client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + query[:query_range] %= context + client_query_range(query[:query_range], start: context[:timeframe_start], stop: context[:timeframe_end]) else - client_query(query[:query] % context, time: context[:timeframe_end]) + query[:query] %= context + client_query(query[:query], time: context[:timeframe_end]) end - query_with_result[:result] = result&.map(&:deep_symbolize_keys) - query_with_result + + query[:result] = result&.map(&:deep_symbolize_keys) + query end def available_metrics @@ -67,6 +70,16 @@ module Gitlab result.select { |group| group.metrics.any? } end + + def common_query_context(environment, timeframe_start:, timeframe_end:) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end, + ci_environment_slug: environment.slug, + kube_namespace: environment.project.kubernetes_service&.actual_namespace || '', + environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + } + end end end end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index a4a97236ffc..536765305e1 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -105,9 +105,32 @@ module Gitlab # # Awesome code block # end def command(*command_names, &block) + define_command(CommandDefinition, *command_names, &block) + end + + # Registers a new substitution which is recognizable from body of email or + # comment. + # It accepts aliases and takes a block with the formatted content. + # + # Example: + # + # command :my_substitution, :alias_for_my_substitution do |text| + # "#{text} MY AWESOME SUBSTITUTION" + # end + def substitution(*substitution_names, &block) + define_command(SubstitutionDefinition, *substitution_names, &block) + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] + end + + private + + def define_command(klass, *command_names, &block) name, *aliases = command_names - definition = CommandDefinition.new( + definition = klass.new( name, aliases: aliases, description: @description, @@ -130,10 +153,6 @@ module Gitlab @condition_block = nil @parse_params_block = nil end - - def definition_by_name(name) - command_definitions_by_name[name.to_sym] - end end end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 09576be7156..3ebfa3bd4b8 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -46,6 +46,8 @@ module Gitlab end end + content, commands = perform_substitutions(content, commands) + [content.strip, commands] end @@ -110,6 +112,26 @@ module Gitlab }mx end + def perform_substitutions(content, commands) + return unless content + + substitution_definitions = self.command_definitions.select do |definition| + definition.is_a?(Gitlab::QuickActions::SubstitutionDefinition) + end + + substitution_definitions.each do |substitution| + match_data = substitution.match(content) + if match_data + command = [substitution.name.to_s] + command << match_data[1] unless match_data[1].empty? + commands << command + end + content = substitution.perform_substitution(self, content) + end + + [content, commands] + end + def command_names(opts) command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb new file mode 100644 index 00000000000..032c49ed159 --- /dev/null +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -0,0 +1,24 @@ +module Gitlab + module QuickActions + class SubstitutionDefinition < CommandDefinition + # noop?=>true means these won't get extracted or removed by Gitlab::QuickActions::Extractor#extract_commands + # QuickActions::InterpretService#perform_substitutions handles them separately + def noop? + true + end + + def match(content) + content.match %r{^/#{all_names.join('|')} ?(.*)$} + end + + def perform_substitution(context, content) + return unless content + + all_names.each do |a_name| + content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + end + content + end + end + end +end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb new file mode 100644 index 00000000000..ccfe0d6bed3 --- /dev/null +++ b/lib/gitlab/request_forgery_protection.rb @@ -0,0 +1,39 @@ +# A module to check CSRF tokens in requests. +# It's used in API helpers and OmniAuth. +# Usage: GitLab::RequestForgeryProtection.call(env) + +module Gitlab + module RequestForgeryProtection + class Controller < ActionController::Base + protect_from_forgery with: :exception + + rescue_from ActionController::InvalidAuthenticityToken do |e| + logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" + logger.warn "Unlike the logs may suggest, this does not result in an actual 422 response to the user" + logger.warn "For API requests, the only effect is that `current_user` will be `nil` for the duration of the request" + + raise e + end + + def index + head :ok + end + end + + def self.app + @app ||= Controller.action(:index) + end + + def self.call(env) + app.call(env) + end + + def self.verified?(env) + call(env) + + true + rescue ActionController::InvalidAuthenticityToken + false + end + end +end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index e71eb15d604..93e00ab75a1 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -21,29 +21,34 @@ module Gitlab from = match[:from] to = match[:to] - actions = find_actions(from, to) + action = find_action(from, to) - if actions.none? - Gitlab::SlashCommands::Presenters::Deploy.new(nil).no_actions - elsif actions.one? - action = play!(from, to, actions.first) - Gitlab::SlashCommands::Presenters::Deploy.new(action).present(from, to) + if action.nil? + Gitlab::SlashCommands::Presenters::Deploy + .new(action).action_not_found else - Gitlab::SlashCommands::Presenters::Deploy.new(actions).too_many_actions + deployment = action.play(current_user) + + Gitlab::SlashCommands::Presenters::Deploy + .new(deployment).present(from, to) end end private - def play!(from, to, action) - action.play(current_user) - end - - def find_actions(from, to) + def find_action(from, to) environment = project.environments.find_by(name: from) - return [] unless environment + return unless environment - environment.actions_for(to).select(&:starts_environment?) + actions = environment.actions_for(to).select do |action| + action.starts_environment? + end + + if actions.many? + actions.find { |action| action.name == to.to_s } + else + actions.first + end end end end diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb index b8dc77bd37b..ebae0f57f9b 100644 --- a/lib/gitlab/slash_commands/presenters/deploy.rb +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -3,17 +3,14 @@ module Gitlab module Presenters class Deploy < Presenters::Base def present(from, to) - message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + message = "Deployment started from #{from} to #{to}. " \ + "[Follow its progress](#{resource_url})." in_channel_response(text: message) end - def no_actions - ephemeral_response(text: "No action found to be executed") - end - - def too_many_actions - ephemeral_response(text: "Too many actions defined") + def action_not_found + ephemeral_response(text: "Couldn't find a deployment manual action.") end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 8e91ee7287c..d9a5af09f08 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -37,8 +37,8 @@ module Gitlab request_cache def can_create_tag?(ref) return false unless can_access_git? - if ProtectedTag.protected?(project, ref) - project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + if protected?(ProtectedTag, project, ref) + protected_tag_accessible_to?(ref, action: :create) else user.can?(:push_code, project) end @@ -47,20 +47,24 @@ module Gitlab request_cache def can_delete_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) user.can?(:delete_protected_branch, project) else user.can?(:push_code, project) end end + def can_update_branch?(ref) + can_push_to_branch?(ref) || can_merge_to_branch?(ref) + end + request_cache def can_push_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) + protected_branch_accessible_to?(ref, action: :push) else user.can?(:push_code, project) end @@ -69,8 +73,8 @@ module Gitlab request_cache def can_merge_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) + if protected?(ProtectedBranch, project, ref) + protected_branch_accessible_to?(ref, action: :merge) else user.can?(:push_code, project) end @@ -87,5 +91,23 @@ module Gitlab def can_access_git? user && user.can?(:access_git) end + + def protected_branch_accessible_to?(ref, action:) + ProtectedBranch.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_branches) + end + + def protected_tag_accessible_to?(ref, action:) + ProtectedTag.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_tags) + end + + request_cache def protected?(kind, project, ref) + kind.protected?(project, ref) + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5dd8a38fea2..3f25e463412 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -35,7 +35,10 @@ module Gitlab when 'git_receive_pack' Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + Gitlab::GitalyClient.feature_enabled?( + :post_upload_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) when 'info_refs' true else diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index 3d60618006c..d80cd7d2a4e 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -24,6 +24,10 @@ module Mattermost json_response session.post(path, options) end + def delete(session, path, options) + json_response session.delete(path, options) + end + def session_get(path, options = {}) with_session do |session| get(session, path, options) @@ -36,6 +40,12 @@ module Mattermost end end + def session_delete(path, options = {}) + with_session do |session| + delete(session, path, options) + end + end + def json_response(response) json_response = JSON.parse(response.body) diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 2cdbbdece16..b2511f3af1d 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -14,5 +14,12 @@ module Mattermost type: type }.to_json) end + + # The deletion is done async, so the response is fast. + # On the mattermost side, this triggers an soft deletion first, after which + # the actuall data is removed + def destroy(team_id:) + session_delete("/api/v4/teams/#{team_id}?permanent=true") + end end end diff --git a/lib/omni_auth/request_forgery_protection.rb b/lib/omni_auth/request_forgery_protection.rb deleted file mode 100644 index 69155131d8d..00000000000 --- a/lib/omni_auth/request_forgery_protection.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Protects OmniAuth request phase against CSRF. - -module OmniAuth - module RequestForgeryProtection - class Controller < ActionController::Base - protect_from_forgery with: :exception - - def index - head :ok - end - end - - def self.app - @app ||= Controller.action(:index) - end - - def self.call(env) - app.call(env) - end - end -end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f93336346..2f2de083dc0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -291,7 +291,7 @@ start_gitlab() { fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The GitLab Workhorse is already running with pid $spid, not restarting" + echo "The GitLab Workhorse is already running with pid $hpid, not restarting" else # No need to remove a socket, gitlab-workhorse does this itself. # Because gitlab-workhorse has multiple executables we need to fix @@ -313,7 +313,7 @@ start_gitlab() { if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages is already running with pid $spid, not restarting" + echo "The GitLab Pages is already running with pid $gppid, not restarting" else $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ @@ -421,7 +421,7 @@ print_status() { fi if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages with pid $mpid is running." + echo "The GitLab Pages with pid $gppid is running." else printf "The GitLab Pages is \033[31mnot running\033[0m.\n" fi diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 003d57adbbd..259a755d724 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -4,6 +4,7 @@ namespace :gitlab do task compile: [ 'yarn:check', 'rake:assets:precompile', + 'gettext:po_to_json', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 858f1cd7b34..dbb3b827b9a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -527,7 +527,7 @@ namespace :gitlab do repo_dirs = user.authorized_projects.map do |p| File.join( p.repository_storage_path, - "#{p.path_with_namespace}.git" + "#{p.disk_path}.git" ) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index a8db5701d0b..3703f9cfb5c 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -19,7 +19,10 @@ namespace :gitlab do Dir.chdir(args.dir) do create_gitaly_configuration - run_command!([command]) + # In CI we run scripts/gitaly-test-build instead of this command + unless ENV['CI'].present? + Bundler.with_original_env { run_command!([command]) } + end end end @@ -30,7 +33,9 @@ namespace :gitlab do puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" puts "# This is in TOML format suitable for use in Gitaly's config.toml file." - puts gitaly_configuration_toml + # Exclude gitaly-ruby configuration because that depends on the gitaly + # installation directory. + puts gitaly_configuration_toml(gitaly_ruby: false) end private @@ -41,7 +46,7 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. - def gitaly_configuration_toml + def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -60,6 +65,7 @@ namespace :gitlab do end config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby TOML.dump(config) end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index ffcc76e5498..b732db9db6e 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(project.repository_storage_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.disk_path) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ee2cdcdea1b..42825f29e32 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -80,7 +80,7 @@ namespace :gitlab do print '-' else if Gitlab::Shell.new.add_repository(project.repository_storage_path, - project.path_with_namespace) + project.disk_path) print '.' else print 'F' diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 4108cee08b4..9cc986535e1 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') +require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' @@ -12,5 +13,6 @@ task setup_postgresql: :environment do AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up IndexRoutesPathForLike.new.up + AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up end |