diff options
| author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:03:02 +0100 |
|---|---|---|
| committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:03:02 +0100 |
| commit | b1f8d8a1739ff48412c8205f0007a2af8399d097 (patch) | |
| tree | f7d35d158e7c9bdda6c282f916e02fe9a0d4df90 /lib | |
| parent | 52c3b8f31264230814d2ffa79d0987c1491676b3 (diff) | |
| parent | 5b08d59f07fc53c1e34819fac20352119d5343e6 (diff) | |
| download | gitlab-ce-b1f8d8a1739ff48412c8205f0007a2af8399d097.tar.gz | |
Merge commit '5b08d59f07fc53c1e34819fac20352119d5343e6' into object-storage-ee-to-ce-backport
Diffstat (limited to 'lib')
218 files changed, 4363 insertions, 2028 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index c9b5f58c557..cdacf9839e5 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -68,6 +68,7 @@ module API delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) + status 204 ::Members::DestroyService.new(source, current_user, params) .execute(:requesters) end diff --git a/lib/api/api.rb b/lib/api/api.rb index efcf0976a81..94df543853b 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,7 @@ module API include APIGuard allow_access_with_scope :api + prefix :api version %w(v3 v4), using: :path @@ -47,8 +48,8 @@ module API end before { header['X-Frame-Options'] = 'SAMEORIGIN' } - before { Gitlab::I18n.locale = current_user&.preferred_language } + # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } rescue_from Gitlab::Access::AccessDeniedError do @@ -85,12 +86,16 @@ module API helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers + NO_SLASH_URL_PART_REGEX = %r{[^/]+} + PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages + mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses mount ::API::DeployKeys @@ -109,7 +114,8 @@ module API mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::Milestones + mount ::API::ProjectMilestones + mount ::API::GroupMilestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -118,6 +124,7 @@ module API mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets + mount ::API::ProtectedBranches mount ::API::Repositories mount ::API::Runner mount ::API::Runners @@ -134,6 +141,7 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::GroupVariables mount ::API::Version route :any, '*path' do diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 56f19f89642..5a028fc9d0b 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -88,6 +88,7 @@ module API unauthorized! unless award.user == current_user || current_user.admin? + status 204 award.destroy end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 3d816f8771d..d3dbf941298 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -4,19 +4,21 @@ module API class Branches < Grape::API include PaginationParams + BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository branches' do success Entities::RepoBranch end params do use :pagination end - get ":id/repository/branches" do + get ':id/repository/branches' do branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) present paginate(branches), with: Entities::RepoBranch, project: user_project @@ -28,13 +30,14 @@ module API params do requires :branch, type: String, desc: 'The name of the branch' end - get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do + get ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do branch = user_project.repository.find_branch(params[:branch]) not_found!("Branch") unless branch present branch, with: Entities::RepoBranch, project: user_project end + # Note: This API will be deprecated in favor of the protected branches API. # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. @@ -46,7 +49,7 @@ module API optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end - put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do + put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -63,9 +66,9 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = if protected_branch - ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + ::ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) else - ProtectedBranches::ApiCreateService.new(*service_args).execute + ::ProtectedBranches::ApiCreateService.new(*service_args).execute end if protected_branch.valid? @@ -75,13 +78,14 @@ module API end end + # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do success Entities::RepoBranch end params do requires :branch, type: String, desc: 'The name of the branch' end - put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do + put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -99,7 +103,7 @@ module API requires :branch, type: String, desc: 'The name of the branch' requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' end - post ":id/repository/branches" do + post ':id/repository/branches' do authorize_push_project result = CreateBranchService.new(user_project, current_user) @@ -118,7 +122,7 @@ module API params do requires :branch, type: String, desc: 'The name of the branch' end - delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do + delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project result = DeleteBranchService.new(user_project, current_user) @@ -130,7 +134,7 @@ module API end desc 'Delete all merged branches' - delete ":id/repository/merged_branches" do + delete ':id/repository/merged_branches' do DeleteMergedBranchesService.new(user_project, current_user).async_execute accepted! diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 395c401203c..9980aec4752 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -91,6 +91,7 @@ module API delete ':id' do message = find_message + status 204 message.destroy end end diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb new file mode 100644 index 00000000000..118883f5ea5 --- /dev/null +++ b/lib/api/circuit_breakers.rb @@ -0,0 +1,50 @@ +module API + class CircuitBreakers < Grape::API + before { authenticated_as_admin! } + + resource :circuit_breakers do + params do + requires :type, + type: String, + desc: "The type of circuitbreaker", + values: ['repository_storage'] + end + resource ':type' do + namespace '', requirements: { type: 'repository_storage' } do + helpers do + def failing_storage_health + @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages + end + + def storage_health + @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages + end + end + + desc 'Get all failing git storages' do + detail 'This feature was introduced in GitLab 9.5' + success Entities::RepositoryStorageHealth + end + get do + present storage_health, with: Entities::RepositoryStorageHealth + end + + desc 'Get all failing git storages' do + detail 'This feature was introduced in GitLab 9.5' + success Entities::RepositoryStorageHealth + end + get 'failing' do + present failing_storage_health, with: Entities::RepositoryStorageHealth + end + + desc 'Reset all storage failures and open circuitbreaker' do + detail 'This feature was introduced in GitLab 9.5' + end + delete do + Gitlab::Git::Storage::CircuitBreaker.reset_all! + end + end + end + end + end +end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index bcb842b9211..ea78737288a 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -4,13 +4,14 @@ module API class Commits < Grape::API include PaginationParams - before { authenticate! } + COMMIT_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: API::NO_SLASH_URL_PART_REGEX) + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do success Entities::RepoCommit end @@ -21,7 +22,7 @@ module API optional :path, type: String, desc: 'The file path' use :pagination end - get ":id/repository/commits" do + get ':id/repository/commits' do path = params[:path] before = params[:until] after = params[:since] @@ -53,16 +54,19 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'The name of branch' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' end - post ":id/repository/commits" do + post ':id/repository/commits' do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) + attrs = declared_params + attrs[:branch_name] = attrs.delete(:branch) + attrs[:start_branch] ||= attrs[:branch_name] result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -76,42 +80,42 @@ module API desc 'Get a specific commit of a project' do success Entities::RepoCommitDetail - failure [[404, 'Not Found']] + failure [[404, 'Commit Not Found']] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha" do + get ':id/repository/commits/:sha', requirements: COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) - not_found! "Commit" unless commit + not_found! 'Commit' unless commit present commit, with: Entities::RepoCommitDetail end desc 'Get the diff for a specific commit of a project' do - failure [[404, 'Not Found']] + failure [[404, 'Commit Not Found']] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha/diff" do + get ':id/repository/commits/:sha/diff', requirements: COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) - not_found! "Commit" unless commit + not_found! 'Commit' unless commit commit.raw_diffs.to_a end desc "Get a commit's comments" do success Entities::CommitNote - failure [[404, 'Not Found']] + failure [[404, 'Commit Not Found']] end params do use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/comments' do + get ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -125,10 +129,10 @@ module API success Entities::RepoCommit end params do - requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' requires :branch, type: String, desc: 'The name of the branch' end - post ':id/repository/commits/:sha/cherry_pick' do + post ':id/repository/commits/:sha/cherry_pick', requirements: COMMIT_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project commit = user_project.commit(params[:sha]) @@ -157,7 +161,7 @@ module API success Entities::CommitNote end params do - requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to post a comment' requires :note, type: String, desc: 'The text of the comment' optional :path, type: String, desc: 'The file path' given :path do @@ -165,7 +169,7 @@ module API requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end - post ':id/repository/commits/:sha/comments' do + post ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index d5c2f3d5094..42e7c1486b0 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -125,6 +125,7 @@ module API key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless key + status 204 key.destroy end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f4796f311a5..6ba4005dd0b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -66,13 +66,6 @@ module API expose :job_events end - class BasicProjectDetails < Grape::Entity - expose :id - expose :http_url_to_repo, :web_url - expose :name, :name_with_namespace - expose :path, :path_with_namespace - end - class SharedGroup < Grape::Entity expose :group_id expose :group_name do |group_link, options| @@ -81,13 +74,51 @@ module API expose :group_access, as: :group_access_level end - class Project < Grape::Entity + class BasicProjectDetails < Grape::Entity expose :id, :description, :default_branch, :tag_list - expose :archived?, as: :archived - expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url - expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } + expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :star_count, :forks_count + expose :created_at, :last_activity_at + end + + class Project < BasicProjectDetails + include ::API::Helpers::RelatedResourcesHelpers + + expose :_links do + expose :self do |project| + expose_url(api_v4_projects_path(id: project.id)) + end + + expose :issues, if: -> (*args) { issues_available?(*args) } do |project| + expose_url(api_v4_projects_issues_path(id: project.id)) + end + + expose :merge_requests, if: -> (*args) { mrs_available?(*args) } do |project| + expose_url(api_v4_projects_merge_requests_path(id: project.id)) + end + + expose :repo_branches do |project| + expose_url(api_v4_projects_repository_branches_path(id: project.id)) + end + + expose :labels do |project| + expose_url(api_v4_projects_labels_path(id: project.id)) + end + + expose :events do |project| + expose_url(api_v4_projects_events_path(id: project.id)) + end + + expose :members do |project| + expose_url(api_v4_projects_members_path(id: project.id)) + end + end + + expose :archived?, as: :archived + expose :visibility + expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible @@ -97,7 +128,6 @@ module API expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } - expose :created_at, :last_activity_at expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id @@ -108,8 +138,7 @@ module API expose :avatar_url do |user, options| user.avatar_url(only_path: false) end - expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs expose :ci_config_path @@ -208,7 +237,7 @@ module API end expose :protected do |repo_branch, options| - ProtectedBranch.protected?(options[:project], repo_branch.name) + ::ProtectedBranch.protected?(options[:project], repo_branch.name) end expose :developers_can_push do |repo_branch, options| @@ -267,10 +296,23 @@ module API expose :deleted_file?, as: :deleted_file end + class ProtectedRefAccess < Grape::Entity + expose :access_level + expose :access_level_description do |protected_ref_access| + protected_ref_access.humanize + end + end + + class ProtectedBranch < Grape::Entity + expose :name + expose :push_access_levels, using: Entities::ProtectedRefAccess + expose :merge_access_levels, using: Entities::ProtectedRefAccess + end + class Milestone < Grape::Entity expose :id, :iid - expose(:project_id) { |entity| entity&.project_id } - expose(:group_id) { |entity| entity&.group_id } + expose :project_id, if: -> (entity, options) { entity&.project_id } + expose :group_id, if: -> (entity, options) { entity&.group_id } expose :title, :description expose :state, :created_at, :updated_at expose :due_date @@ -297,6 +339,26 @@ module API end class Issue < IssueBasic + include ::API::Helpers::RelatedResourcesHelpers + + expose :_links do + expose :self do |issue| + expose_url(api_v4_project_issue_path(id: issue.project_id, issue_iid: issue.iid)) + end + + expose :notes do |issue| + expose_url(api_v4_projects_issues_notes_path(id: issue.project_id, noteable_id: issue.iid)) + end + + expose :award_emoji do |issue| + expose_url(api_v4_projects_issues_award_emoji_path(id: issue.project_id, issue_iid: issue.iid)) + end + + expose :project do |issue| + expose_url(api_v4_projects_path(id: issue.project_id)) + end + end + expose :subscribed do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end @@ -314,12 +376,35 @@ module API expose :id end + class MergeRequestSimple < ProjectEntity + expose :title + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end + end + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].upvotes + else + merge_request.upvotes + end + end + expose :downvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].downvotes + else + merge_request.downvotes + end + end expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :label_names, as: :labels + expose :labels do |merge_request, options| + # Avoids an N+1 query since labels are preloaded + merge_request.labels.map(&:title).sort + end expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds @@ -369,6 +454,9 @@ module API end class Note < Grape::Entity + # Only Issue and MergeRequest have iid + NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze + expose :id expose :note, as: :body expose :attachment_identifier, as: :attachment @@ -376,6 +464,9 @@ module API expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type + + # Avoid N+1 queries as much as possible + expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) } end class AwardEmoji < Grape::Entity @@ -408,7 +499,7 @@ module API class Event < Grape::Entity expose :title, :project_id, :action_name - expose :target_id, :target_type, :author_id + expose :target_id, :target_iid, :target_type, :author_id expose :data, :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } @@ -596,42 +687,14 @@ module API class ApplicationSetting < Grape::Entity expose :id - expose :default_projects_limit - expose :signup_enabled - expose :signin_enabled - expose :gravatar_enabled - expose :sign_in_text - expose :after_sign_up_text - expose :created_at - expose :updated_at - expose :home_page_url - expose :default_branch_protection + expose(*::ApplicationSettingsHelper.visible_attributes) expose(:restricted_visibility_levels) do |setting, _options| setting.restricted_visibility_levels.map { |level| Gitlab::VisibilityLevel.string_level(level) } end - expose :max_attachment_size - expose :session_expire_delay expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } - expose :default_artifacts_expire_in - expose :domain_whitelist - expose :domain_blacklist_enabled - expose :domain_blacklist - expose :user_oauth_applications - expose :after_sign_out_path - expose :container_registry_token_expire_delay - expose :repository_storage - expose :repository_storages - expose :koding_enabled - expose :koding_url - expose :plantuml_enabled - expose :plantuml_url - expose :terminal_max_session_time - expose :polling_interval_multiplier - expose :help_page_hide_commercial_content - expose :help_page_text - expose :help_page_support_url + expose :password_authentication_enabled, as: :signin_enabled end class Release < Grape::Entity @@ -642,7 +705,7 @@ module API class RepoTag < Grape::Entity expose :name, :message - expose :commit do |repo_tag, options| + expose :commit, using: Entities::RepoCommit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) end @@ -894,5 +957,11 @@ module API expose :ip_address expose :submitted, as: :akismet_submitted end + + class RepositoryStorageHealth < Grape::Entity + expose :storage_name + expose :failing_on_hosts + expose :total_failures + end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 945771d46f3..c774a5c6685 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -79,6 +79,7 @@ module API environment = user_project.environments.find(params[:environment_id]) + status 204 environment.destroy end diff --git a/lib/api/files.rb b/lib/api/files.rb index 521287ee2b4..450334fee84 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -4,7 +4,7 @@ module API def commit_params(attrs) { file_path: attrs[:file_path], - start_branch: attrs[:branch], + start_branch: attrs[:start_branch] || attrs[:branch], branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], @@ -37,8 +37,9 @@ module API params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :branch, type: String, desc: 'The name of branch' - requires :commit_message, type: String, desc: 'Commit Message' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :commit_message, type: String, desc: 'Commit message' + optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'The email of the author' optional :author_name, type: String, desc: 'The name of the author' end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb new file mode 100644 index 00000000000..b85eb59dc0a --- /dev/null +++ b/lib/api/group_milestones.rb @@ -0,0 +1,85 @@ +module API + class GroupMilestones < Grape::API + include MilestoneResponses + include PaginationParams + + before do + authenticate! + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Get a list of group milestones' do + success Entities::Milestone + end + params do + use :list_params + end + get ":id/milestones" do + list_milestones_for(user_group) + end + + desc 'Get a single group milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + end + get ":id/milestones/:milestone_id" do + authorize! :read_group, user_group + + get_milestone_for(user_group) + end + + desc 'Create a new group milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end + post ":id/milestones" do + authorize! :admin_milestones, user_group + + create_milestone_for(user_group) + end + + desc 'Update an existing group milestone' do + success Entities::Milestone + end + params do + use :update_params + end + put ":id/milestones/:milestone_id" do + authorize! :admin_milestones, user_group + + update_milestone_for(user_group) + end + + desc 'Get all issues for a single group milestone' do + success Entities::IssueBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + use :pagination + end + get ":id/milestones/:milestone_id/issues" do + milestone_issuables_for(user_group, :issue) + end + + desc 'Get all merge requests for a single group milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequestBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a group milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + milestone_issuables_for(user_group, :merge_request) + end + end + end +end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb new file mode 100644 index 00000000000..f64da4ab77b --- /dev/null +++ b/lib/api/group_variables.rb @@ -0,0 +1,96 @@ +module API + class GroupVariables < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize! :admin_build, user_group } + + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Get group-level variables' do + success Entities::Variable + end + params do + use :pagination + end + get ':id/variables' do + variables = user_group.variables + present paginate(variables), with: Entities::Variable + end + + desc 'Get a specific variable from a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + get ':id/variables/:key' do + key = params[:key] + variable = user_group.variables.find_by(key: key) + + return not_found!('GroupVariable') unless variable + + present variable, with: Entities::Variable + end + + desc 'Create a new variable in a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' + end + post ':id/variables' do + variable_params = declared_params(include_missing: false) + + variable = user_group.variables.create(variable_params) + + if variable.valid? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Update an existing variable from a group' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' + end + put ':id/variables/:key' do + variable = user_group.variables.find_by(key: params[:key]) + + return not_found!('GroupVariable') unless variable + + variable_params = declared_params(include_missing: false).except(:key) + + if variable.update(variable_params) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Delete an existing variable from a group' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/variables/:key' do + variable = user_group.variables.find_by(key: params[:key]) + not_found!('GroupVariable') unless variable + + status 204 + variable.destroy + end + end + end +end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ebbaed0cbb7..49c3b2278c7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -125,6 +125,8 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group + + status 204 ::Groups::DestroyService.new(group, current_user).execute end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 7003390113b..4056d06bcc7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -16,6 +16,8 @@ module API @current_user = initial_current_user + Gitlab::I18n.locale = @current_user&.preferred_language + sudo! @current_user @@ -25,6 +27,10 @@ module API initial_current_user != current_user end + def user_group + @group ||= find_group!(params[:id]) + end + def user_project @project ||= find_project!(params[:id]) end @@ -332,12 +338,14 @@ module API env['warden'] end + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(env) + end + # Check the Rails session for valid authentication details - # - # Until CSRF protection is added to the API, disallow this method for - # state-changing endpoints def find_user_from_warden - warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) + warden.try(:authenticate) if verified_request? end def initial_current_user diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb new file mode 100644 index 00000000000..1f677529b07 --- /dev/null +++ b/lib/api/helpers/related_resources_helpers.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module RelatedResourcesHelpers + include GrapeRouteHelpers::NamedRouteMatcher + + def issues_available?(project, options) + available?(:issues, project, options[:current_user]) + end + + def mrs_available?(project, options) + available?(:merge_requests, project, options[:current_user]) + end + + def expose_url(path) + url_options = Gitlab::Application.routes.default_url_options + protocol, host, port = url_options.slice(:protocol, :host, :port).values + + URI::HTTP.build(scheme: protocol, host: host, port: port, path: path).to_s + end + + private + + def available?(feature, project, current_user) + project.feature_available?(feature, current_user) + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ef2c08e902c..8b007869dc3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -101,7 +101,7 @@ module API end get "/broadcast_message" do - if message = BroadcastMessage.current.last + if message = BroadcastMessage.current&.last present message, with: Entities::BroadcastMessage else {} @@ -150,7 +150,7 @@ module API # # begin # repository = wiki? ? project.wiki.repository : project.repository - # Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive + # Gitlab::GitalyClient::NotificationService.new(repository.raw_repository).post_receive # rescue GRPC::Unavailable => e # render_api_error!(e, 500) # end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 64be08094ed..4cec1145f3a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -29,6 +29,10 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' + optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], + desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' use :pagination end @@ -55,9 +59,11 @@ module API optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' use :issues_params + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', + desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do - issues = find_issues(scope: 'authored') + issues = find_issues present paginate(issues), with: Entities::IssueBasic, current_user: current_user end @@ -112,7 +118,7 @@ module API params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - get ":id/issues/:issue_iid" do + get ":id/issues/:issue_iid", as: :api_v4_project_issue do issue = find_project_issue(params[:issue_iid]) present issue, with: Entities::Issue, current_user: current_user, project: user_project end @@ -224,6 +230,7 @@ module API not_found!('Issue') unless issue authorize!(:destroy_issue, issue) + status 204 issue.destroy end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 20b25529d0c..4520c98d951 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -56,6 +56,7 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label + status 204 label.destroy end diff --git a/lib/api/members.rb b/lib/api/members.rb index c200e46a328..bb970b7cd54 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -96,6 +96,7 @@ module API # Ensure that memeber exists source.members.find_by!(user_id: params[:user_id]) + status 204 ::Members::DestroyService.new(source, current_user, declared_params).execute end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index d419d345ec5..8810d4e441d 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -4,6 +4,71 @@ module API before { authenticate! } + helpers ::Gitlab::IssuableMetadata + + helpers do + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute + .reorder(args[:order_by] => args[:sort]) + merge_requests = paginate(merge_requests) + .preload(:target_project) + + return merge_requests if args[:view] == 'simple' + + merge_requests + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) + end + + params :merge_requests_params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' + optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' + optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' + optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], + desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + use :pagination + end + end + + resource :merge_requests do + desc 'List merge requests' do + success Entities::MergeRequestBasic + end + params do + use :merge_requests_params + optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', + desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + end + get do + merge_requests = find_merge_requests + + options = { with: Entities::MergeRequestBasic, + current_user: current_user } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -27,27 +92,6 @@ module API render_api_error!(errors, 400) end - def issue_entity(project) - if project.has_external_issue_tracker? - Entities::ExternalIssue - else - Entities::IssueBasic - end - end - - def find_merge_requests(args = {}) - args = params.merge(args) - - args[:milestone_title] = args.delete(:milestone) - args[:label_name] = args.delete(:labels) - - merge_requests = MergeRequestsFinder.new(current_user, args).execute - .inc_notes_with_associations - .preload(:target_project, :author, :assignee, :milestone, :merge_request_diff) - - merge_requests.reorder(args[:order_by] => args[:sort]) - end - params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -65,25 +109,25 @@ module API success Entities::MergeRequestBasic end params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return merge requests sorted in `asc` or `desc` order.' + use :merge_requests_params optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' - optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' - optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' - optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' - use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project merge_requests = find_merge_requests(project_id: user_project.id) - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + options = { with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options end desc 'Create a merge request' do @@ -120,6 +164,7 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) + status 204 merge_request.destroy end @@ -260,7 +305,14 @@ module API get ':id/merge_requests/:merge_request_iid/closes_issues' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: issue_entity(user_project), current_user: current_user + issues = paginate(issues) + + external_issues, internal_issues = issues.partition { |issue| issue.is_a?(ExternalIssue) } + + data = Entities::IssueBasic.represent(internal_issues, current_user: current_user) + data += Entities::ExternalIssue.represent(external_issues, current_user: current_user) + + data.as_json end end end diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb new file mode 100644 index 00000000000..ef09d9505d2 --- /dev/null +++ b/lib/api/milestone_responses.rb @@ -0,0 +1,98 @@ +module API + module MilestoneResponses + extend ActiveSupport::Concern + + included do + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + end + + params :list_params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' + optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' + use :pagination + end + + params :update_params do + requires :milestone_id, type: Integer, desc: 'The milestone ID number' + optional :title, type: String, desc: 'The title of the milestone' + optional :state_event, type: String, values: %w[close activate], + desc: 'The state event of the milestone ' + use :optional_params + at_least_one_of :title, :description, :due_date, :state_event + end + + def list_milestones_for(parent) + milestones = parent.milestones + milestones = Milestone.filter_by_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? + milestones = filter_by_search(milestones, params[:search]) if params[:search] + + present paginate(milestones), with: Entities::Milestone + end + + def get_milestone_for(parent) + milestone = parent.milestones.find(params[:milestone_id]) + present milestone, with: Entities::Milestone + end + + def create_milestone_for(parent) + milestone = ::Milestones::CreateService.new(parent, current_user, declared_params).execute + + if milestone.valid? + present milestone, with: Entities::Milestone + else + render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) + end + end + + def update_milestone_for(parent) + milestone = parent.milestones.find(params.delete(:milestone_id)) + + milestone_params = declared_params(include_missing: false) + milestone = ::Milestones::UpdateService.new(parent, current_user, milestone_params).execute(milestone) + + if milestone.valid? + present milestone, with: Entities::Milestone + else + render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) + end + end + + def milestone_issuables_for(parent, type) + milestone = parent.milestones.find(params[:milestone_id]) + + finder_klass, entity = get_finder_and_entity(type) + + params = build_finder_params(milestone, parent) + + issuables = finder_klass.new(current_user, params).execute + present paginate(issuables), with: entity, current_user: current_user + end + + def build_finder_params(milestone, parent) + finder_params = { milestone_title: milestone.title, sort: 'label_priority' } + + if parent.is_a?(Group) + finder_params.merge(group_id: parent.id) + else + finder_params.merge(project_id: parent.id) + end + end + + def get_finder_and_entity(type) + if type == :issue + [IssuesFinder, Entities::IssueBasic] + else + [MergeRequestsFinder, Entities::MergeRequestBasic] + end + end + end + end + end +end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb deleted file mode 100644 index 3541d3c95fb..00000000000 --- a/lib/api/milestones.rb +++ /dev/null @@ -1,154 +0,0 @@ -module API - class Milestones < Grape::API - include PaginationParams - - before { authenticate! } - - helpers do - def filter_milestones_state(milestones, state) - case state - when 'active' then milestones.active - when 'closed' then milestones.closed - else milestones - end - end - - params :optional_params do - optional :description, type: String, desc: 'The description of the milestone' - optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' - optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a list of project milestones' do - success Entities::Milestone - end - params do - optional :state, type: String, values: %w[active closed all], default: 'all', - desc: 'Return "active", "closed", or "all" milestones' - optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' - optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' - use :pagination - end - get ":id/milestones" do - authorize! :read_milestone, user_project - - milestones = user_project.milestones - milestones = filter_milestones_state(milestones, params[:state]) - milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? - milestones = filter_by_search(milestones, params[:search]) if params[:search] - - present paginate(milestones), with: Entities::Milestone - end - - desc 'Get a single project milestone' do - success Entities::Milestone - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - end - get ":id/milestones/:milestone_id" do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - present milestone, with: Entities::Milestone - end - - desc 'Create a new project milestone' do - success Entities::Milestone - end - params do - requires :title, type: String, desc: 'The title of the milestone' - use :optional_params - end - post ":id/milestones" do - authorize! :admin_milestone, user_project - - milestone = ::Milestones::CreateService.new(user_project, current_user, declared_params).execute - - if milestone.valid? - present milestone, with: Entities::Milestone - else - render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) - end - end - - desc 'Update an existing project milestone' do - success Entities::Milestone - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - optional :title, type: String, desc: 'The title of the milestone' - optional :state_event, type: String, values: %w[close activate], - desc: 'The state event of the milestone ' - use :optional_params - at_least_one_of :title, :description, :due_date, :state_event - end - put ":id/milestones/:milestone_id" do - authorize! :admin_milestone, user_project - milestone = user_project.milestones.find(params.delete(:milestone_id)) - - milestone_params = declared_params(include_missing: false) - milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone) - - if milestone.valid? - present milestone, with: Entities::Milestone - else - render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) - end - end - - desc 'Get all issues for a single project milestone' do - success Entities::IssueBasic - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - use :pagination - end - get ":id/milestones/:milestone_id/issues" do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - - finder_params = { - project_id: user_project.id, - milestone_title: milestone.title, - sort: 'label_priority' - } - - issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project - end - - desc 'Get all merge requests for a single project milestone' do - detail 'This feature was introduced in GitLab 9.' - success Entities::MergeRequestBasic - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - use :pagination - end - get ':id/milestones/:milestone_id/merge_requests' do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - - finder_params = { - project_id: user_project.id, - milestone_title: milestone.title, - sort: 'label_priority' - } - - merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute - present paginate(merge_requests), - with: Entities::MergeRequestBasic, - current_user: current_user, - project: user_project - end - end - end -end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 01ca62b593f..65ff89edf65 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -131,6 +131,7 @@ module API note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note + status 204 ::Notes::DestroyService.new(user_project, current_user).execute(note) end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 7a345289617..649dd891f56 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -96,6 +96,7 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) + status 204 hook.destroy end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb new file mode 100644 index 00000000000..451998c726a --- /dev/null +++ b/lib/api/project_milestones.rb @@ -0,0 +1,91 @@ +module API + class ProjectMilestones < Grape::API + include PaginationParams + include MilestoneResponses + + before do + authenticate! + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a list of project milestones' do + success Entities::Milestone + end + params do + use :list_params + end + get ":id/milestones" do + authorize! :read_milestone, user_project + + list_milestones_for(user_project) + end + + desc 'Get a single project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end + get ":id/milestones/:milestone_id" do + authorize! :read_milestone, user_project + + get_milestone_for(user_project) + end + + desc 'Create a new project milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end + post ":id/milestones" do + authorize! :admin_milestone, user_project + + create_milestone_for(user_project) + end + + desc 'Update an existing project milestone' do + success Entities::Milestone + end + params do + use :update_params + end + put ":id/milestones/:milestone_id" do + authorize! :admin_milestone, user_project + + update_milestone_for(user_project) + end + + desc 'Get all issues for a single project milestone' do + success Entities::IssueBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ":id/milestones/:milestone_id/issues" do + authorize! :read_milestone, user_project + + milestone_issuables_for(user_project, :issue) + end + + desc 'Get all merge requests for a single project milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequestBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + authorize! :read_milestone, user_project + + milestone_issuables_for(user_project, :merge_request) + end + end + end +end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 3320eadff0d..f3d905b0068 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -116,6 +116,7 @@ module API not_found!('Snippet') unless snippet authorize! :admin_project_snippet, snippet + status 204 snippet.destroy end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index c459257158d..89dda88d3f5 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -361,6 +361,7 @@ module API authorize! :remove_fork_project, user_project if user_project.forked? + status 204 user_project.forked_project_link.destroy else not_modified! @@ -405,6 +406,7 @@ module API link = user_project.project_group_links.find_by(group_id: params[:group_id]) not_found!('Group Link') unless link + status 204 link.destroy end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb new file mode 100644 index 00000000000..d742f2e18d0 --- /dev/null +++ b/lib/api/protected_branches.rb @@ -0,0 +1,85 @@ +module API + class ProtectedBranches < Grape::API + include PaginationParams + + BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_admin_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a project's protected branches" do + success Entities::ProtectedBranch + end + params do + use :pagination + end + get ':id/protected_branches' do + protected_branches = user_project.protected_branches.preload(:push_access_levels, :merge_access_levels) + + present paginate(protected_branches), with: Entities::ProtectedBranch, project: user_project + end + + desc 'Get a single protected branch' do + success Entities::ProtectedBranch + end + params do + requires :name, type: String, desc: 'The name of the branch or wildcard' + end + get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + protected_branch = user_project.protected_branches.find_by!(name: params[:name]) + + present protected_branch, with: Entities::ProtectedBranch, project: user_project + end + + desc 'Protect a single branch or wildcard' do + success Entities::ProtectedBranch + end + params do + requires :name, type: String, desc: 'The name of the protected branch' + optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, + values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + desc: 'Access levels allowed to push (defaults: `40`, master access level)' + optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, + values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + desc: 'Access levels allowed to merge (defaults: `40`, master access level)' + end + post ':id/protected_branches' do + protected_branch = user_project.protected_branches.find_by(name: params[:name]) + if protected_branch + conflict!("Protected branch '#{params[:name]}' already exists") + end + + protected_branch_params = { + name: params[:name], + push_access_levels_attributes: [{ access_level: params[:push_access_level] }], + merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] + } + + service_args = [user_project, current_user, protected_branch_params] + + protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + + if protected_branch.persisted? + present protected_branch, with: Entities::ProtectedBranch, project: user_project + else + render_api_error!(protected_branch.errors.full_messages, 422) + end + end + + desc 'Unprotect a single branch' + params do + requires :name, type: String, desc: 'The name of the protected branch' + end + delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + protected_branch = user_project.protected_branches.find_by!(name: params[:name]) + + protected_branch.destroy + + status 204 + end + end + end +end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 4552115b3e2..88fc62d33df 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -45,6 +45,7 @@ module API end delete '/' do authenticate_runner! + status 204 Ci::Runner.find_by_token(params[:token]).destroy end @@ -89,7 +90,7 @@ module API if result.valid? if result.build Gitlab::Metrics.add_event(:build_found, - project: result.build.project.path_with_namespace) + project: result.build.project.full_path) present result.build, with: Entities::JobRequest::Response else Gitlab::Metrics.add_event(:build_not_found) @@ -118,7 +119,7 @@ module API job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, - project: job.project.path_with_namespace) + project: job.project.full_path) case params[:state].to_s when 'success' diff --git a/lib/api/runners.rb b/lib/api/runners.rb index db6c7c59092..5bf5a18e42f 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -79,6 +79,7 @@ module API runner = get_runner(params[:id]) authenticate_delete_runner!(runner) + status 204 runner.destroy! end end @@ -134,6 +135,7 @@ module API runner = runner_project.runner forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 + status 204 runner_project.destroy end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 7488f95a9b7..843c05ae32e 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -313,12 +313,6 @@ module API desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { - required: true, - name: :project_key, - type: String, - desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' - }, - { required: false, name: :username, type: String, diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d598f9a62a2..d55a61fa638 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -20,58 +20,6 @@ module API success Entities::ApplicationSetting end params do - # CE - at_least_one_of_ce = [ - :admin_notification_email, - :after_sign_out_path, - :after_sign_up_text, - :akismet_enabled, - :container_registry_token_expire_delay, - :default_artifacts_expire_in, - :default_branch_protection, - :default_group_visibility, - :default_project_visibility, - :default_projects_limit, - :default_snippet_visibility, - :disabled_oauth_sign_in_sources, - :domain_blacklist_enabled, - :domain_whitelist, - :email_author_in_body, - :enabled_git_access_protocol, - :gravatar_enabled, - :help_page_hide_commercial_content, - :help_page_text, - :help_page_support_url, - :home_page_url, - :housekeeping_enabled, - :html_emails_enabled, - :import_sources, - :koding_enabled, - :max_artifacts_size, - :max_attachment_size, - :max_pages_size, - :metrics_enabled, - :plantuml_enabled, - :polling_interval_multiplier, - :recaptcha_enabled, - :repository_checks_enabled, - :repository_storage, - :require_two_factor_authentication, - :restricted_visibility_levels, - :send_user_confirmation_email, - :sentry_enabled, - :clientside_sentry_enabled, - :session_expire_delay, - :shared_runners_enabled, - :sidekiq_throttling_enabled, - :sign_in_text, - :signin_enabled, - :signup_enabled, - :terminal_max_session_time, - :user_default_external, - :user_oauth_applications, - :version_check_enabled - ] optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' @@ -95,7 +43,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -148,7 +98,7 @@ module API given clientside_sentry_enabled: ->(val) { val } do requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' end - optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :koding_enabled, type: Boolean, desc: 'Enable Koding' given koding_enabled: ->(val) { val } do @@ -171,11 +121,16 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' - at_least_one_of(*at_least_one_of_ce) + optional(*::ApplicationSettingsHelper.visible_attributes) + at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) end put "application/settings" do attrs = declared_params(include_missing: false) + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index fd634037a77..35ece56c65c 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -123,6 +123,7 @@ module API authorize! :destroy_personal_snippet, snippet + status 204 snippet.destroy end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index ed7b23b474a..c0179037440 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -66,6 +66,7 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook + status 204 hook.destroy end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 633a858f8c7..1333747cced 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -2,19 +2,21 @@ module API class Tags < Grape::API include PaginationParams + TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository tags' do success Entities::RepoTag end params do use :pagination end - get ":id/repository/tags" do + get ':id/repository/tags' do tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) present paginate(tags), with: Entities::RepoTag, project: user_project end @@ -25,7 +27,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag' end - get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -60,7 +62,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag' end - delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project result = ::Tags::DestroyService.new(user_project, current_user) @@ -78,7 +80,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag' requires :description, type: String, desc: 'Release notes with markdown support' end - post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do + post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project result = CreateReleaseService.new(user_project, current_user) @@ -98,7 +100,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag' requires :description, type: String, desc: 'Release notes with markdown support' end - put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do + put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project result = UpdateReleaseService.new(user_project, current_user) diff --git a/lib/api/todos.rb b/lib/api/todos.rb index d1f7e364029..55191169dd4 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -59,10 +59,10 @@ module API requires :id, type: Integer, desc: 'The ID of the todo being marked as done' end post ':id/mark_as_done' do - todo = current_user.todos.find(params[:id]) - TodoService.new.mark_todos_as_done([todo], current_user) + TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) + todo = Todo.find(params[:id]) - present todo.reload, with: Entities::Todo, current_user: current_user + present todo, with: Entities::Todo, current_user: current_user end desc 'Mark all todos as done' diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index a9f2ca2608e..edfdb63d183 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,24 +15,22 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do - project = find_project(params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) - if trigger_request - present trigger_request.pipeline, with: Entities::Pipeline + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute + not_found! unless result + + if result[:http_status] + render_api_error!(result[:message], result[:http_status]) else - errors = 'No pipeline created' - render_api_error!(errors, 400) + present result[:pipeline], with: Entities::Pipeline end end @@ -142,6 +140,7 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger + status 204 trigger.destroy end end diff --git a/lib/api/users.rb b/lib/api/users.rb index c469751c31c..a590f2692a2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -235,6 +235,7 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + status 204 key.destroy end @@ -306,6 +307,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user + status 204 user.delete_async(deleted_by: current_user, params: params) end @@ -406,6 +408,7 @@ module API requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end delete ':impersonation_token_id' do + status 204 find_impersonation_token.revoke! end end @@ -421,7 +424,16 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic + entity = + if sudo? + Entities::UserWithPrivateDetails + elsif current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end + + present current_user, with: entity end desc "Get the currently authenticated user's SSH keys" do @@ -474,6 +486,7 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + status 204 key.destroy end @@ -525,6 +538,7 @@ module API email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email + status 204 Emails::DestroyService.new(current_user, email: email.email).execute end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c848f52723b..773f667abe0 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -161,7 +161,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text @@ -258,11 +259,40 @@ module API expose :job_events, as: :build_events end - class Issue < ::API::Entities::Issue + class ProjectEntity < Grape::Entity + expose :id, :iid + expose(:project_id) { |entity| entity&.project.try(:id) } + expose :title, :description + expose :state, :created_at, :updated_at + end + + class IssueBasic < ProjectEntity + expose :label_names, as: :labels + expose :milestone, using: ::API::Entities::Milestone + expose :assignees, :author, using: ::API::Entities::UserBasic + + expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + issue.assignees.first + end + + expose :user_notes_count + expose :upvotes, :downvotes + expose :due_date + expose :confidential + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end + end + + class Issue < IssueBasic unexpose :assignees expose :assignee do |issue, options| ::API::Entities::UserBasic.represent(issue.assignees.first, options) end + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user], options[:project] || issue.project) + end end end end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb index 94614bfc8b6..51014591a93 100644 --- a/lib/api/v3/project_hooks.rb +++ b/lib/api/v3/project_hooks.rb @@ -56,7 +56,9 @@ module API use :project_hook_properties end post ":id/hooks" do - hook = user_project.hooks.new(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) + hook = user_project.hooks.new(attrs) if hook.save present hook, with: ::API::V3::Entities::ProjectHook @@ -77,7 +79,9 @@ module API put ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - if hook.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) + if hook.update_attributes(attrs) present hook, with: ::API::V3::Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 748d6b97d4f..202011cfcbe 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,7 +44,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -116,7 +118,7 @@ module API :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, :user_oauth_applications, :user_default_external, :signup_enabled, :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication, :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, @@ -126,7 +128,13 @@ module API :housekeeping_enabled, :terminal_max_session_time end put "application/settings" do - if current_settings.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb index e3b311d61cd..2f2cf259987 100644 --- a/lib/api/v3/todos.rb +++ b/lib/api/v3/todos.rb @@ -11,10 +11,10 @@ module API requires :id, type: Integer, desc: 'The ID of the todo being marked as done' end delete ':id' do - todo = current_user.todos.find(params[:id]) - TodoService.new.mark_todos_as_done([todo], current_user) + TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) + todo = Todo.find(params[:id]) - present todo.reload, with: ::API::Entities::Todo, current_user: current_user + present todo, with: ::API::Entities::Todo, current_user: current_user end desc 'Mark all todos as done' diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index a23d6b6b48c..e9d4c35307b 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -28,12 +28,13 @@ module API end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) - if trigger_request - present trigger_request, with: ::API::V3::Entities::TriggerRequest + result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref].to_s, variables) + pipeline = result.pipeline + + if pipeline.persisted? + present result.trigger_request, with: ::API::V3::Entities::TriggerRequest else - errors = 'No builds created' - render_api_error!(errors, 400) + render_validation_error!(pipeline) end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 7fa528fb2d3..7c0fdd3d1be 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -88,6 +88,7 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable + status 204 variable.destroy end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f755c99ea4a..ca6d6848d41 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -8,18 +8,9 @@ module Backup # Make sure there is a connection ActiveRecord::Base.connection.reconnect! - # saving additional informations - s = {} - s[:db_version] = "#{ActiveRecord::Migrator.current_version}" - s[:backup_created_at] = Time.now - s[:gitlab_version] = Gitlab::VERSION - s[:tar_version] = tar_version - s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(backup_path) do File.open("#{backup_path}/backup_information.yml", "w+") do |file| - file << s.to_yaml.gsub(/^---\n/, '') + file << backup_information.to_yaml.gsub(/^---\n/, '') end # create archive @@ -33,11 +24,11 @@ module Backup abort 'Backup failed' end - upload(tar_file) + upload end end - def upload(tar_file) + def upload $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection @@ -48,7 +39,7 @@ module Backup directory = connect_to_remote_directory(connection_settings) - if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, + if directory.files.create(key: remote_target, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, storage_class: Gitlab.config.backup.upload.storage_class) @@ -177,7 +168,8 @@ module Backup end def connect_to_remote_directory(connection_settings) - connection = ::Fog::Storage.new(connection_settings) + # our settings use string keys, but Fog expects symbols + connection = ::Fog::Storage.new(connection_settings.symbolize_keys) # We only attempt to create the directory for local backups. For AWS # and other cloud providers, we cannot guarantee the user will have @@ -193,6 +185,14 @@ module Backup Gitlab.config.backup.upload.remote_directory end + def remote_target + if ENV['DIRECTORY'] + File.join(ENV['DIRECTORY'], tar_file) + else + tar_file + end + end + def backup_contents folders_to_backup + archives_to_backup + ["backup_information.yml"] end @@ -214,5 +214,19 @@ module Backup def settings @settings ||= YAML.load_file("backup_information.yml") end + + def tar_file + @tar_file ||= "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}" + end + + def backup_information + @backup_information ||= { + db_version: ActiveRecord::Migrator.current_version.to_s, + backup_created_at: Time.now, + gitlab_version: Gitlab::VERSION, + tar_version: tar_version, + skipped: ENV["SKIP"] + } + end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index a1685c77916..88821ae56e0 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,7 +7,7 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.full_path} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -42,7 +42,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.full_path} ... " if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else @@ -71,11 +71,11 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.full_path} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_dir_exist + project.ensure_storage_path_exist cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) @@ -104,7 +104,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.full_path} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -142,11 +142,11 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + '.bundle') + File.join(backup_repos_path, project.disk_path + '.bundle') end def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.path_with_namespace) + path = File.join(backup_repos_path, project.disk_path) if dir File.join(path, "#{dir}.tar") @@ -185,13 +185,14 @@ module Backup def progress_warn(project, cmd, output) progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + progress.puts "Ignoring error on #{project.full_path} - #{output}".color(:orange) end def empty_repo?(project_or_wiki) + project_or_wiki.repository.expire_exists_cache # protect backups from stale cache project_or_wiki.repository.empty_repo? rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange) + progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.full_path} - #{e.message}".color(:orange) false end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 7a262dd025c..ef4578aabd6 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -54,42 +54,42 @@ module Banzai self.class.references_in(*args, &block) end + # Implement in child class + # Example: project.merge_requests.find def find_object(project, id) - # Implement in child class - # Example: project.merge_requests.find end - def find_object_cached(project, id) - if RequestStore.active? - cache = find_objects_cache[object_class][project.id] + # Override if the link reference pattern produces a different ID (global + # ID vs internal ID, for instance) to the regular reference pattern. + def find_object_from_link(project, id) + find_object(project, id) + end - get_or_set_cache(cache, id) { find_object(project, id) } - else + # Implement in child class + # Example: project_merge_request_url + def url_for_object(object, project) + end + + def find_object_cached(project, id) + cached_call(:banzai_find_object, id, path: [object_class, project.id]) do find_object(project, id) end end - def project_from_ref_cached(ref) - if RequestStore.active? - cache = project_refs_cache - - get_or_set_cache(cache, ref) { project_from_ref(ref) } - else - project_from_ref(ref) + def find_object_from_link_cached(project, id) + cached_call(:banzai_find_object_from_link, id, path: [object_class, project.id]) do + find_object_from_link(project, id) end end - def url_for_object(object, project) - # Implement in child class - # Example: project_merge_request_url + def project_from_ref_cached(ref) + cached_call(:banzai_project_refs, ref) do + project_from_ref(ref) + end end def url_for_object_cached(object, project) - if RequestStore.active? - cache = url_for_object_cache[object_class][project.id] - - get_or_set_cache(cache, object) { url_for_object(object, project) } - else + cached_call(:banzai_url_for_object, object, path: [object_class, project.id]) do url_for_object(object, project) end end @@ -120,7 +120,7 @@ module Banzai if link == inner_html && inner_html =~ /\A#{link_pattern}/ replace_link_node_with_text(node, link) do - object_link_filter(inner_html, link_pattern) + object_link_filter(inner_html, link_pattern, link_reference: true) end next @@ -128,7 +128,7 @@ module Banzai if link =~ /\A#{link_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, link_pattern, link_content: inner_html) + object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true) end next @@ -146,15 +146,26 @@ module Banzai # text - String text to replace references in. # pattern - Reference pattern to match against. # link_content - Original content of the link being replaced. + # link_reference - True if this was using the link reference pattern, + # false otherwise. # # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text, pattern, link_content: nil) + def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| project_path = full_project_path(namespace_ref, project_ref) project = project_from_ref_cached(project_path) - if project && object = find_object_cached(project, id) + if project + object = + if link_reference + find_object_from_link_cached(project, id) + else + find_object_cached(project, id) + end + end + + if object title = object_link_title(object) klass = reference_class(object_sym) @@ -259,7 +270,7 @@ module Banzai found = [] projects.each do |project| - ref = project.path_with_namespace + ref = project.full_path get_or_set_cache(cache, ref) { project } found << ref end @@ -277,7 +288,7 @@ module Banzai end def current_project_path - @current_project_path ||= project.path_with_namespace + @current_project_path ||= project.full_path end def current_project_namespace_path @@ -297,15 +308,17 @@ module Banzai RequestStore[:banzai_project_refs] ||= {} end - def find_objects_cache - RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key| - hash[key] = Hash.new { |h, k| h[k] = {} } - end - end + def cached_call(request_store_key, cache_key, path: []) + if RequestStore.active? + cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end - def url_for_object_cache - RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key| - hash[key] = Hash.new { |h, k| h[k] = {} } + cache = cache.dig(*path) if path.any? + + get_or_set_cache(cache, cache_key) { yield } + else + yield end end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 0ea4eeaed5b..2e259904673 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -118,7 +118,7 @@ module Banzai end if path - content_tag(:img, nil, src: path, class: 'gfm') + content_tag(:img, nil, data: { src: path }, class: 'gfm') end end diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb new file mode 100644 index 00000000000..7a81d583b82 --- /dev/null +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -0,0 +1,16 @@ +module Banzai + module Filter + # HTML filter that moves the value of the src attribute to the data-src attribute so it can be lazy loaded + class ImageLazyLoadFilter < HTML::Pipeline::Filter + def call + doc.xpath('descendant-or-self::img').each do |img| + img['class'] ||= '' << 'lazy' + img['data-src'] = img['src'] + img['src'] = LazyImageTagHelper.placeholder_image + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index 123c92fd250..f318c425962 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -10,7 +10,7 @@ module Banzai link = doc.document.create_element( 'a', class: 'no-attachment-icon', - href: img['src'], + href: img['data-src'] || img['src'], target: '_blank', rel: 'noopener noreferrer' ) diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index ba1a5ac84b3..ce1ab977d3b 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -20,7 +20,7 @@ module Banzai end def url_for_object(issue, project) - IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) + IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true) end def project_from_ref(ref) diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 45c033d32a8..4fc5f211e84 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -8,8 +8,15 @@ module Banzai Milestone end + # Links to project milestones contain the IID, but when we're handling + # 'regular' references, we need to use the global ID to disambiguate + # between group and project milestones. def find_object(project, id) - project.milestones.find_by(iid: id) + find_milestone_with_finder(project, id: id) + end + + def find_object_from_link(project, iid) + find_milestone_with_finder(project, iid: iid) end def references_in(text, pattern = Milestone.reference_pattern) @@ -22,7 +29,7 @@ module Banzai milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone - yield match, milestone.iid, $~[:project], $~[:namespace], $~ + yield match, milestone.id, $~[:project], $~[:namespace], $~ else match end @@ -36,7 +43,8 @@ module Banzai return unless project milestone_params = milestone_params(milestone_id, milestone_name) - project.milestones.find_by(milestone_params) + + find_milestone_with_finder(project, milestone_params) end def milestone_params(iid, name) @@ -47,15 +55,27 @@ module Banzai end end + def find_milestone_with_finder(project, params) + finder_params = { project_ids: [project.id], order: nil } + + # We don't support IID lookups for group milestones, because IIDs can + # clash between group and project milestones. + if project.group && !params[:iid] + finder_params[:group_ids] = [project.group.id] + end + + MilestonesFinder.new(finder_params).execute.find_by(params) + end + def url_for_object(milestone, project) - h = Gitlab::Routing.url_helpers - h.project_milestone_url(project, milestone, - only_path: context[:only_path]) + Gitlab::Routing + .url_helpers + .milestone_url(milestone, only_path: context[:only_path]) end def object_link_text(object, matches) milestone_link = escape_once(super) - reference = object.project.to_reference(project) + reference = object.project&.to_reference(project) if reference.present? "#{milestone_link} <i>in #{reference}</i>".html_safe diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 9e23c8f8c55..758f15c8a67 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -22,6 +22,7 @@ module Banzai doc.css('img, video').each do |el| process_link_attr el.attribute('src') + process_link_attr el.attribute('data-src') end doc @@ -50,7 +51,7 @@ module Banzai uri.path = [ relative_url_root, - context[:project].path_with_namespace, + context[:project].full_path, uri_type(file_path), Addressable::URI.escape(ref), Addressable::URI.escape(file_path) diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 45bb66dc99f..09844931be5 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -28,7 +28,7 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) + File.join(Gitlab.config.gitlab.url, project.full_path, uri) end def project diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index bd4d1aa9ff8..3208abfc538 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -16,6 +16,7 @@ module Banzai Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, + Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, Filter::EmojiFilter, Filter::TableOfContentsFilter, diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index 6307c1b571a..1802cd04854 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -21,10 +21,14 @@ module Banzai gather_attributes_per_project(nodes, self.class.data_attribute) end - private - + # we extract only external issue trackers references here, we don't extract cross-project references, + # so we don't need to do anything here. def can_read_reference?(user, ref_project, node) - can?(user, :read_issue, ref_project) + true + end + + def nodes_visible_to_user(user, nodes) + nodes end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7801cb5baf..ad08c0905e2 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -132,6 +132,8 @@ module Banzai end def self.cacheless_render(text, context = {}) + return text.to_s unless text.present? + Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index ecbdfb3e4b2..812ba8ed603 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -29,7 +29,7 @@ module Ci if result.valid? if result.build Gitlab::Metrics.add_event(:build_found, - project: result.build.project.path_with_namespace) + project: result.build.project.full_path) present result.build, with: Entities::BuildDetails else @@ -64,7 +64,7 @@ module Ci build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, - project: build.project.path_with_namespace) + project: build.project.full_path) case params[:state].to_s when 'success' diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 6b82b2b4f13..31f66dd5a58 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -52,7 +52,7 @@ module Ci # when old API will be removed (planned for August 2017). model.options.dup.tap do |options| options[:image] = options[:image][:name] if options[:image].is_a?(Hash) - options[:services].map! do |service| + options[:services]&.map! do |service| if service.is_a?(Hash) service[:name] else diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb index 6e622601680..6225203f223 100644 --- a/lib/ci/api/triggers.rb +++ b/lib/ci/api/triggers.rb @@ -24,12 +24,13 @@ module Ci end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref], variables) - if trigger_request - present trigger_request, with: Entities::TriggerRequest + result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) + pipeline = result.pipeline + + if pipeline.persisted? + present result.trigger_request, with: Entities::TriggerRequest else - errors = 'No builds created' - render_api_error!(errors, 400) + render_validation_error!(pipeline) end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 56ad2c77c7d..3a4911b23b0 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -80,8 +80,11 @@ module Ci artifacts: job[:artifacts], cache: job[:cache], dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], after_script: job[:after_script], - environment: job[:environment] + environment: job[:environment], + retry: job[:retry] }.compact } end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index d9959bc1aff..ae65653645b 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -8,7 +8,12 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' +require 'thread' + module DeclarativePolicy + CLASS_CACHE_MUTEX = Mutex.new + CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE + class << self def policy_for(user, subject, opts = {}) cache = opts[:cache] || {} @@ -23,7 +28,40 @@ module DeclarativePolicy subject = find_delegate(subject) - subject.class.ancestors.each do |klass| + policy_class = class_for_class(subject.class) + raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class + end + + def has_policy?(subject) + !class_for_class(subject.class).nil? + end + + private + + # This method is heavily cached because there are a lot of anonymous + # modules in play in a typical rails app, and #name performs quite + # slowly for anonymous classes and modules. + # + # See https://bugs.ruby-lang.org/issues/11119 + # + # if the above bug is resolved, this caching could likely be removed. + def class_for_class(subject_class) + unless subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + CLASS_CACHE_MUTEX.synchronize do + # re-check in case of a race + break if subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + + policy_class = compute_class_for_class(subject_class) + subject_class.instance_variable_set(CLASS_CACHE_IVAR, policy_class) + end + end + + subject_class.instance_variable_get(CLASS_CACHE_IVAR) + end + + def compute_class_for_class(subject_class) + subject_class.ancestors.each do |klass| next unless klass.name begin @@ -38,11 +76,9 @@ module DeclarativePolicy end end - raise "no policy for #{subject.class.name}" + nil end - private - def find_delegate(subject) seen = Set.new diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index b8cc60074c7..0804edba016 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -21,11 +21,14 @@ module DeclarativePolicy private def id_for(obj) - if obj.respond_to?(:id) && obj.id - obj.id.to_s - else - "##{obj.object_id}" - end + id = + begin + obj.id + rescue NoMethodError + nil + end + + id || "##{obj.object_id}" end end end diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb index 9d7cf6b9726..51c4a8b2bbe 100644 --- a/lib/declarative_policy/condition.rb +++ b/lib/declarative_policy/condition.rb @@ -82,13 +82,14 @@ module DeclarativePolicy # depending on the scope, we may cache only by the user or only by # the subject, resulting in sharing across different policy objects. def cache_key - case @condition.scope - when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" - when :user then "/dp/condition/#{@condition.key}/#{user_key}" - when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" - when :global then "/dp/condition/#{@condition.key}" - else raise 'invalid scope' - end + @cache_key ||= + case @condition.scope + when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" + when :user then "/dp/condition/#{@condition.key}/#{user_key}" + when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" + when :global then "/dp/condition/#{@condition.key}" + else raise 'invalid scope' + end end def user_key diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index b5c615da4e3..56afd1f1392 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -76,6 +76,8 @@ module DeclarativePolicy @state = State.new steps_by_score do |step, score| + return if !debug && @state.prevented? + passed = nil case step.action when :enable then @@ -93,10 +95,7 @@ module DeclarativePolicy # been prevented. unless @state.prevented? passed = step.pass? - if passed - @state.prevent! - return unless debug - end + @state.prevent! if passed end debug << inspect_step(step, score, passed) if debug @@ -141,13 +140,14 @@ module DeclarativePolicy end steps = Set.new(@steps) + remaining_enablers = steps.count { |s| s.enable? } loop do return if steps.empty? # if the permission hasn't yet been enabled and we only have # prevent steps left, we short-circuit the state here - @state.prevent! if !@state.enabled? && steps.all?(&:prevent?) + @state.prevent! if !@state.enabled? && remaining_enablers == 0 lowest_score = Float::INFINITY next_step = nil @@ -162,6 +162,8 @@ module DeclarativePolicy steps.delete(next_step) + remaining_enablers -= 1 if next_step.enable? + yield next_step, lowest_score end end diff --git a/lib/feature.rb b/lib/feature.rb index 363f66ba60e..4bd29aed687 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -57,5 +57,11 @@ class Feature Flipper.new(adapter) end end + + # This method is called from config/initializers/flipper.rb and can be used + # to register Flipper groups. + # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups + def register_feature_groups + end end end diff --git a/lib/github/client.rb b/lib/github/client.rb index e65d908d232..9c476df7d46 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,13 +1,16 @@ module Github class Client + TIMEOUT = 60 + attr_reader :connection, :rate_limit def initialize(options) - @connection = Faraday.new(url: options.fetch(:url)) do |faraday| - faraday.options.open_timeout = options.fetch(:timeout, 60) - faraday.options.timeout = options.fetch(:timeout, 60) + @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT) + faraday.options.timeout = options.fetch(:timeout, TIMEOUT) faraday.authorization 'token', options.fetch(:token) faraday.adapter :net_http + faraday.ssl.verify = verify_ssl end @rate_limit = RateLimit.new(connection) @@ -19,5 +22,32 @@ module Github Github::Response.new(connection.get(url, query)) end + + private + + def root_endpoint + custom_endpoint || github_endpoint + end + + def custom_endpoint + github_omniauth_provider.dig('args', 'client_options', 'site') + end + + def verify_ssl + # If there is no config, we're connecting to github.com + # and we should verify ssl. + github_omniauth_provider.fetch('verify_ssl', true) + end + + def github_endpoint + OmniAuth::Strategies::GitHub.default_options[:client_options][:site] + end + + def github_omniauth_provider + @github_omniauth_provider ||= + Gitlab.config.omniauth.providers + .find { |provider| provider.name == 'github' } + .to_h + end end end diff --git a/lib/github/import.rb b/lib/github/import.rb index ff5d7db2705..4cc01593ef4 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -41,13 +41,16 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose + attr_reader :project, :repository, :repo, :repo_url, :wiki_url, + :options, :errors, :cached, :verbose - def initialize(project, options) + def initialize(project, options = {}) @project = project @repository = project.repository @repo = project.import_source - @options = options + @repo_url = project.import_url + @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') + @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user)) @verbose = options.fetch(:verbose, false) @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] @@ -65,6 +68,8 @@ module Github fetch_pull_requests puts 'Fetching issues...'.color(:aqua) if verbose fetch_issues + puts 'Fetching releases...'.color(:aqua) if verbose + fetch_releases puts 'Cloning wiki repository...'.color(:aqua) if verbose fetch_wiki_repository puts 'Expiring repository cache...'.color(:aqua) if verbose @@ -72,6 +77,7 @@ module Github true rescue Github::RepositoryFetchError + expire_repository_cache false ensure keep_track_of_errors @@ -81,23 +87,21 @@ module Github def fetch_repository begin - project.create_repository unless project.repository.exists? - project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git") + project.ensure_repository + project.repository.add_remote('github', repo_url) project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) - rescue Gitlab::Shell::Error => e - error(:project, "https://github.com/#{repo}.git", e.message) + rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + error(:project, repo_url, e.message) raise Github::RepositoryFetchError end end def fetch_wiki_repository - wiki_url = "https://#{options.fetch(:token)}@github.com/#{repo}.wiki.git" - wiki_path = "#{project.path_with_namespace}.wiki" + return if project.wiki.repository_exists? - unless project.wiki.repository_exists? - gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) - end + wiki_path = "#{project.disk_path}.wiki" + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, @@ -309,7 +313,7 @@ module Github next unless representation.valid? release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) - next unless relese.new_record? + next unless release.new_record? begin release.description = representation.description @@ -337,7 +341,7 @@ module Github def user_id(user, fallback_id = nil) return unless user.present? - return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) + return cached[:user_ids][user.id] if cached[:user_ids][user.id].present? gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) @@ -367,7 +371,7 @@ module Github end def expire_repository_cache - repository.expire_content_cache + repository.expire_content_cache if project.repository_exists? end def keep_track_of_errors diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ccb5d886bab..7d3aa532750 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,7 +37,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -48,6 +48,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -214,7 +218,8 @@ module Gitlab def full_authentication_abilities read_authentication_abilities + [ :push_code, - :create_container_image + :create_container_image, + :admin_container_image ] end alias_method :api_scope_authentication_abilities, :full_authentication_abilities diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d95ecd7b291..d3f66877672 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,24 +1,45 @@ module Gitlab module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue - .new(BackgroundMigrationWorker.sidekiq_options['queue']) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - queue.each do |job| - migration_class, migration_args = job.args + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args - next unless migration_class == steal_class + next unless job.queue == self.queue + next unless migration_class == steal_class - perform(migration_class, migration_args) + begin + perform(migration_class, migration_args) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) - job.delete + raise + end + end end end + ## + # Performs a background migration. + # # class_name - The name of the background migration class as defined in the # Gitlab::BackgroundMigration namespace. # diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb new file mode 100644 index 00000000000..0fbc6b70989 --- /dev/null +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -0,0 +1,107 @@ +module Gitlab + module BackgroundMigration + class DeserializeMergeRequestDiffsAndCommits + attr_reader :diff_ids, :commit_rows, :file_rows + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + BUFFER_ROWS = 1000 + + def perform(start_id, stop_id) + merge_request_diffs = MergeRequestDiff + .select(:id, :st_commits, :st_diffs) + .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') + .where(id: start_id..stop_id) + + reset_buffers! + + merge_request_diffs.each do |merge_request_diff| + commits, files = single_diff_rows(merge_request_diff) + + diff_ids << merge_request_diff.id + commit_rows.concat(commits) + file_rows.concat(files) + + if diff_ids.length > BUFFER_ROWS || + commit_rows.length > BUFFER_ROWS || + file_rows.length > BUFFER_ROWS + + flush_buffers! + end + end + + flush_buffers! + end + + private + + def reset_buffers! + @diff_ids = [] + @commit_rows = [] + @file_rows = [] + end + + def flush_buffers! + if diff_ids.any? + MergeRequestDiff.transaction do + Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) + Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + end + end + + reset_buffers! + end + + def single_diff_rows(merge_request_diff) + sha_attribute = Gitlab::Database::ShaAttribute.new + commits = YAML.load(merge_request_diff.st_commits) rescue [] + + commit_rows = commits.map.with_index do |commit, index| + commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) + sha = commit_hash.delete(:id) + + commit_hash.merge( + merge_request_diff_id: merge_request_diff.id, + relative_order: index, + sha: sha_attribute.type_cast_for_database(sha) + ) + end + + diffs = YAML.load(merge_request_diff.st_diffs) rescue [] + diffs = [] unless valid_raw_diffs?(diffs) + + file_rows = diffs.map.with_index do |diff, index| + diff_hash = diff.to_hash.with_indifferent_access.merge( + binary: false, + merge_request_diff_id: merge_request_diff.id, + relative_order: index + ) + + # Compatibility with old diffs created with Psych. + diff_hash.tap do |hash| + diff_text = hash[:diff] + + if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? + hash[:binary] = true + hash[:diff] = [diff_text].pack('m0') + end + end + end + + [commit_rows, file_rows] + end + + # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as + # valid, because we don't render them usefully anyway. + def valid_raw_diffs?(diffs) + return false unless diffs.respond_to?(:each) + + diffs.all? { |diff| diff.is_a?(Hash) } + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb new file mode 100644 index 00000000000..0881244ed49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -0,0 +1,26 @@ +module Gitlab + module BackgroundMigration + class MigrateSystemUploadsToNewFolder + include Gitlab::Database::MigrationHelpers + attr_reader :old_folder, :new_folder + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + include EachBatch + end + + def perform(old_folder, new_folder) + replace_sql = replace_sql(uploads[:path], old_folder, new_folder) + affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) + + affected_uploads.each_batch do |batch| + batch.update_all("path = #{replace_sql}") + end + end + + def uploads + Arel::Table.new('uploads') + end + end + end +end diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/pipeline/metadata.rb index 2ee35a0d4c1..db1e9f8cfb8 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/pipeline/metadata.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that describes build badge metadata + # Class that describes pipeline badge metadata # class Metadata < Badge::Metadata def initialize(badge) @@ -11,11 +11,11 @@ module Gitlab end def title - 'build status' + 'pipeline status' end def image_url - build_project_badges_url(@project, @ref, format: :svg) + pipeline_project_badges_url(@project, @ref, format: :svg) end def link_url diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/pipeline/status.rb index b762d85b6e5..5fee7a93475 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Build status badge + # Pipeline status badge # class Status < Badge::Base attr_reader :project, :ref @@ -15,7 +15,7 @@ module Gitlab end def entity - 'build' + 'pipeline' end def status @@ -25,11 +25,11 @@ module Gitlab end def metadata - @metadata ||= Build::Metadata.new(self) + @metadata ||= Pipeline::Metadata.new(self) end def template - @template ||= Build::Template.new(self) + @template ||= Pipeline::Template.new(self) end end end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/pipeline/template.rb index bc0e0cd441d..e09db32262d 100644 --- a/lib/gitlab/badge/build/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -1,12 +1,13 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that represents a build badge template. + # Class that represents a pipeline badge template. # # Template object will be passed to badge.svg.erb template. # class Template < Badge::Template + STATUS_RENAME = { 'success' => 'passed' }.freeze STATUS_COLOR = { success: '#4c1', failed: '#e05d44', @@ -27,11 +28,11 @@ module Gitlab end def value_text - @status.to_s + STATUS_RENAME[@status.to_s] || @status.to_s end def key_width - 38 + 62 end def value_width diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 5a6d9ae99a0..28bbf3b384e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,7 +61,7 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.path_with_namespace}.wiki" + path_with_namespace = "#{project.full_path}.wiki" import_url = project.import_url.sub(/\.git\z/, ".git/wiki") gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) rescue StandardError => e diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb new file mode 100644 index 00000000000..f1a04affd38 --- /dev/null +++ b/lib/gitlab/cache/request_cache.rb @@ -0,0 +1,94 @@ +module Gitlab + module Cache + # This module provides a simple way to cache values in RequestStore, + # and the cache key would be based on the class name, method name, + # optionally customized instance level values, optionally customized + # method level values, and optional method arguments. + # + # A simple example: + # + # class UserAccess + # extend Gitlab::Cache::RequestCache + # + # request_cache_key do + # [user&.id, project&.id] + # end + # + # request_cache def can_push_to_branch?(ref) + # # ... + # end + # end + # + # This way, the result of `can_push_to_branch?` would be cached in + # `RequestStore.store` based on the cache key. If RequestStore is not + # currently active, then it would be stored in a hash saved in an + # instance variable, so the cache logic would be the same. + # Here's another example using customized method level values: + # + # class Commit + # extend Gitlab::Cache::RequestCache + # + # def author + # User.find_by_any_email(author_email.downcase) + # end + # request_cache(:author) { author_email.downcase } + # end + # + # So that we could have different strategies for different methods + # + module RequestCache + def self.extended(klass) + return if klass < self + + extension = Module.new + klass.const_set(:RequestCacheExtension, extension) + klass.prepend(extension) + end + + def request_cache_key(&block) + if block_given? + @request_cache_key = block + else + @request_cache_key + end + end + + def request_cache(method_name, &method_key_block) + const_get(:RequestCacheExtension).module_eval do + cache_key_method_name = "#{method_name}_cache_key" + + define_method(method_name) do |*args| + store = + if RequestStore.active? + RequestStore.store + else + ivar_name = # ! and ? cannot be used as ivar name + "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" + + instance_variable_get(ivar_name) || + instance_variable_set(ivar_name, {}) + end + + key = __send__(cache_key_method_name, args) + + store.fetch(key) { store[key] = super(*args) } + end + + define_method(cache_key_method_name) do |args| + klass = self.class + + instance_key = instance_exec(&klass.request_cache_key) if + klass.request_cache_key + + method_key = instance_exec(&method_key_block) if method_key_block + + [klass.name, method_name, *instance_key, *method_key, *args] + .join(':') + end + + private cache_key_method_name + end + end + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 176301bcca1..32f5c6ab142 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment coverage].freeze + after_script variables environment coverage retry].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -23,6 +23,9 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true + validates :retry, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ @@ -76,9 +79,9 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage + :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies + attributes :script, :tags, :allow_failure, :when, :dependencies, :retry def compose!(deps = nil) super do @@ -142,6 +145,7 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, + retry: retry_defined? ? retry_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c4c0623df6c..8503ecf8700 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -67,14 +67,15 @@ module Gitlab def extract_coverage(regex) return unless valid? - return unless regex + return unless regex.present? - regex = Regexp.new(regex) + regex = Gitlab::UntrustedRegexp.new(regex) match = "" reverse_line do |line| - matches = line.scan(regex) + line.chomp! + matches = regex.scan(line) next unless matches.is_a?(Array) next if matches.empty? diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index bf557103cfd..0735243e021 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -48,7 +48,7 @@ module Gitlab end def starting_month - Date.today.month + Date.current.month end private @@ -66,12 +66,18 @@ module Gitlab .select(:id) conditions = t[:created_at].gteq(date_from.beginning_of_day) - .and(t[:created_at].lteq(Date.today.end_of_day)) + .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) + date_interval = if Gitlab::Database.postgresql? + "INTERVAL '#{Time.zone.now.utc_offset} seconds'" + else + "INTERVAL #{Time.zone.now.utc_offset} SECOND" + end + Event.reorder(nil) - .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount') - .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)') + .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') + .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") .where(conditions) .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 818b3d9c46b..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available end end @@ -33,12 +33,7 @@ module Gitlab def uncached_application_settings return fake_application_settings unless connect_to_db? - # This loads from the database into the cache, so handle Redis errors - begin - db_settings = ::ApplicationSetting.current - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - end + db_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb new file mode 100644 index 00000000000..dfd17e35707 --- /dev/null +++ b/lib/gitlab/daemon.rb @@ -0,0 +1,62 @@ +module Gitlab + class Daemon + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + Kernel.at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance ||= initialize_instance + end + + attr_reader :thread + + def thread? + !thread.nil? + end + + def initialize + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return thread if thread? + + @thread = Thread.new { start_working } + end + end + + def stop + @mutex.synchronize do + return unless thread? + + stop_working + + if thread + thread.wakeup if thread.alive? + thread.join + @thread = nil + end + end + end + + private + + def start_working + raise NotImplementedError + end + + def stop_working + # no-ops + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index e81d19a7a2e..5c5f507d44d 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -24,11 +24,11 @@ module Gitlab # total_commits_count: Fixnum # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) commits = Array(commits) # Total commits count - commits_count = commits.size + commits_count ||= commits.size # Get latest 20 commits ASC commits_limited = commits.last(20) @@ -74,6 +74,8 @@ module Gitlab build(project, user, commits.last&.id, commits.first&.id, ref, commits) end + private + def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb new file mode 100644 index 00000000000..226974b698c --- /dev/null +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -0,0 +1,22 @@ +module Gitlab + module DataBuilder + module WikiPage + extend self + + def build(wiki_page, user, action) + wiki = wiki_page.wiki + + { + object_kind: wiki_page.class.name.underscore, + user: user.hook_attrs, + project: wiki.project.hook_attrs, + wiki: wiki.hook_attrs, + object_attributes: wiki_page.hook_attrs.merge( + url: Gitlab::UrlBuilder.build(wiki_page), + action: action + ) + } + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0643c56db9b..69ca9aa596b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -140,6 +140,8 @@ module Gitlab return add_foreign_key(source, target, column: column, on_delete: on_delete) + else + on_delete = 'SET NULL' if on_delete == :nullify end disable_statement_timeout @@ -155,7 +157,7 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete}" : ''} + #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} NOT VALID; EOF diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb deleted file mode 100644 index a78fde9d782..00000000000 --- a/lib/gitlab/devise_failure.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - class DeviseFailure < Devise::FailureApp - protected - - # Override `Devise::FailureApp#request_format` to handle a special case - # - # This tells Devise to handle an unauthenticated `.zip` request as an HTML - # request (i.e., redirect to sign in). - # - # Otherwise, Devise would respond with a 401 Unauthorized with - # `Content-Type: application/zip` and a response body in plaintext, and the - # browser would freak out. - # - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 - def request_format - if request.format == :zip - Mime::Type.lookup_by_extension(:html).ref - else - super - end - end - end -end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2863a4da71..6d7de52cb80 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -79,13 +79,6 @@ module Gitlab @new_content_sha = refs&.head_sha end - def new_content_commit - return @new_content_commit if defined?(@new_content_commit) - - sha = new_content_commit - @new_content_commit = repository.commit(sha) if sha - end - def old_content_sha return if new_file? return @old_content_sha if defined?(@old_content_sha) @@ -94,13 +87,6 @@ module Gitlab @old_content_sha = refs&.base_sha end - def old_content_commit - return @old_content_commit if defined?(@old_content_commit) - - sha = old_content_sha - @old_content_commit = repository.commit(sha) if sha - end - def new_blob return @new_blob if defined?(@new_blob) @@ -123,10 +109,6 @@ module Gitlab new_content_sha || old_content_sha end - def content_commit - new_content_commit || old_content_commit - end - def blob new_blob || old_blob end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 1a5887dab7e..72d7d4f84d1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -181,8 +181,6 @@ module Gitlab end def find_merge_base_with_master(branch:) - return if merge_base_found? - # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -237,6 +235,10 @@ module Gitlab branch_name.parameterize << '.patch' end + def patch_url + "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/#{ENV['CI_JOB_ID']}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" + end + def step(desc, cmd = nil) puts "\n=> #{desc}\n" @@ -303,14 +305,11 @@ module Gitlab 2. Apply your branch's patch to EE - # In the CE repo - $ git fetch origin master - $ git diff --binary origin/master...HEAD -- > #{ce_branch}.patch - # In the EE repo $ git fetch origin master $ git checkout -b #{ee_branch_prefix} origin/master - $ git apply --3way path/to/#{ce_branch}.patch + $ wget #{patch_url} + $ git apply --3way #{ce_patch_name} At this point you might have conflicts such as: @@ -324,7 +323,7 @@ module Gitlab If the patch couldn't be applied cleanly, use the following command: # In the EE repo - $ git apply --reject path/to/#{ce_branch}.patch + $ git apply --reject #{ce_patch_name} This option makes git apply the parts of the patch that are applicable, and leave the rejected hunks in corresponding `.rej` files. @@ -337,7 +336,7 @@ module Gitlab # In the EE repo $ git push origin #{ee_branch_prefix} - ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. Once this is done, you can retry this failed build, and it should pass. diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 31579e94a87..8eea33b9ab5 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -15,7 +15,6 @@ module Gitlab def execute raise SentNotificationNotFoundError unless sent_notification - raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/ validate_permission!(:create_note) diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index dd1d9dcd555..cd9d3a6483f 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -117,7 +117,7 @@ module Gitlab def subject subject_text = '[Git]' - subject_text << "[#{project.path_with_namespace}]" + subject_text << "[#{project.full_path}]" subject_text << "[#{ref_name}]" if @action == :push subject_text << ' ' diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 0d6b08b5d29..c8f4591d060 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -26,6 +26,9 @@ module Gitlab raise EmptyEmailError if @raw.blank? mail = build_mail + + ignore_auto_submitted!(mail) + mail_key = extract_mail_key(mail) handler = Handler.for(mail, mail_key) @@ -87,6 +90,16 @@ module Gitlab break key if key end end + + def ignore_auto_submitted!(mail) + # Mail::Header#[] is case-insensitive + auto_submitted = mail.header['Auto-Submitted']&.value + + # Mail::Field#value would strip leading and trailing whitespace + raise AutoGeneratedEmailError if + # See also https://tools.ietf.org/html/rfc3834 + auto_submitted && auto_submitted != 'no' + end end end end diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb new file mode 100644 index 00000000000..5e0dd6e7859 --- /dev/null +++ b/lib/gitlab/environment.rb @@ -0,0 +1,7 @@ +module Gitlab + module Environment + def self.hostname + @hostname ||= ENV['HOSTNAME'] || Socket.gethostname + end + end +end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a0f46594eb1..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -26,17 +26,17 @@ module Gitlab EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -45,24 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4175746be39..b6449f27034 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -10,7 +10,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode! ref.sub(/\Arefs\/(tags|heads)\//, '') + encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..8dbe25e55f6 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -25,13 +25,28 @@ module Gitlab private def load_blame - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) - # Read in binary mode to ensure ASCII-8BIT - raw_output = IO.popen(cmd, 'rb') {|io| io.read } + raw_output = @repo.gitaly_migrate(:blame) do |is_enabled| + if is_enabled + load_blame_by_gitaly + else + load_blame_by_shelling_out + end + end + output = encode_utf8(raw_output) process_raw_blame output end + def load_blame_by_gitaly + @repo.gitaly_commit_client.raw_blame(@sha, @path) + end + + def load_blame_by_shelling_out + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + IO.popen(cmd, 'rb') {|io| io.read } + end + def process_raw_blame(output) lines, final = [], [] info, commits = {}, {} diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ea386c2ddcb..77b81d2d437 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -18,66 +20,7 @@ module Gitlab if is_enabled find_by_gitaly(repository, sha, path) else - find_by_rugged(repository, sha, path) - end - end - end - - def find_by_gitaly(repository, sha, path) - path = path.sub(/\A\/*/, '') - path = '/' if path.empty? - name = File.basename(path) - entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) - return unless entry - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, path) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) + find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) end end end @@ -85,10 +28,10 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else blob = repository.lookup(sha) - + new( id: blob.oid, size: blob.size, @@ -107,6 +50,23 @@ module Gitlab detect && detect[:type] == :binary end + # Returns an array of Blob instances, specified in blob_references as + # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the + # full blob contents are returned. If blob_size_limit >= 0 then each blob will + # contain no more than limit bytes in its data attribute. + # + # Keep in mind that this method may allocate a lot of memory. It is up + # to the caller to limit the number of blobs and blob_size_limit. + # + def batch(repository, blob_references, blob_size_limit: nil) + blob_size_limit ||= MAX_DATA_DISPLAY_SIZE + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end + + private + # Recursive search of blob id by path # # Ex. @@ -149,6 +109,66 @@ module Gitlab commit_id: sha ) end + + def find_by_gitaly(repository, sha, path) + path = path.sub(/\A\/*/, '') + path = '/' if path.empty? + name = File.basename(path) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + return unless entry + + case entry.type + when :COMMIT + new( + id: entry.oid, + name: name, + size: 0, + data: '', + path: path, + commit_id: sha + ) + when :BLOB + new( + id: entry.oid, + name: name, + size: entry.size, + data: entry.data.dup, + mode: entry.mode.to_s(8), + path: path, + commit_id: sha, + binary: binary?(entry.data) + ) + end + end + + def find_by_rugged(repository, sha, path, limit:) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + # Rugged::Blob#content is expensive; don't call it if we don't have to. + data: limit.zero? ? '' : blob.content(limit), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end end def initialize(options) @@ -178,7 +198,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 9c0606d780a..600d886e818 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -97,8 +98,12 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.commits_between(base, head).map do |commit| - decorate(commit) + Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.between(base, head) + else + repo.rugged_commits_between(base, head).map { |c| decorate(c) } + end end rescue Rugged::ReferenceError [] @@ -124,7 +129,18 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) + Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| + if is_enabled + find_all_by_gitaly(repo, options) + else + find_all_by_rugged(repo, options) + end + end + end + + def find_all_by_rugged(repo, options = {}) actual_options = options.dup allowed_options = [:ref, :max_count, :skip, :order] @@ -163,6 +179,10 @@ module Gitlab [] end + def find_all_by_gitaly(repo, options = {}) + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) + end + def decorate(commit, ref = nil) Gitlab::Git::Commit.new(commit, ref) end @@ -204,10 +224,13 @@ module Gitlab def initialize(raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit - if raw_commit.is_a?(Hash) + case raw_commit + when Hash init_from_hash(raw_commit) - elsif raw_commit.is_a?(Rugged::Commit) + when Rugged::Commit init_from_rugged(raw_commit) + when Gitlab::GitalyClient::Commit + init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end @@ -243,6 +266,8 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff diff_from_parent.patch end @@ -284,7 +309,23 @@ module Gitlab end def parents - raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + case raw_commit + when Rugged::Commit + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + when Gitlab::GitalyClient::Commit + parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact + else + raise NotImplementedError, "commit source doesn't support #parents" + end + end + + # Get the gpg signature of this commit. + # + # Ex. + # commit.signature(repo) + # + def signature(repo) + Rugged::Commit.extract_signature(repo.rugged, sha) end def stats @@ -295,7 +336,7 @@ module Gitlab begin raw_commit.to_mbox(options) rescue Rugged::InvalidError => ex - if ex.message =~ /Commit \w+ is a merge commit/ + if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' end end @@ -367,6 +408,22 @@ module Gitlab @parent_ids = commit.parents.map(&:oid) end + def init_from_gitaly(commit) + @raw_commit = commit + @id = commit.id + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + @message = (commit.body.presence || commit.subject).dup + @authored_date = Time.at(commit.author.date.seconds) + @author_name = commit.author.name.dup + @author_email = commit.author.email.dup + @committed_date = Time.at(commit.committer.date.seconds) + @committer_name = commit.committer.name.dup + @committer_email = commit.committer.email.dup + @parent_ids = commit.parent_ids + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 78e440395a5..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index cf7829a583b..9e00abefd02 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -81,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options @@ -326,6 +234,8 @@ module Gitlab @new_file = diff.from_id == BLANK_SHA @renamed_file = diff.from_path != diff.to_path @deleted_file = diff.to_id == BLANK_SHA + + collapse! if diff.respond_to?(:collapsed) && diff.collapsed end def prune_diff_if_eligible diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 555894907cc..87ed9c3ea26 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection @@ -5,16 +7,28 @@ module Gitlab DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + attr_reader :limits + + delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + + def self.collection_limits(options = {}) + limits = {} + limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file + limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + + OpenStruct.new(limits) + end + def initialize(iterator, options = {}) @iterator = iterator - @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file - @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min - @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @limits = self.class.collection_limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) + @from_gitaly = options.fetch(:from_gitaly, false) @line_count = 0 @byte_count = 0 @@ -24,9 +38,23 @@ module Gitlab end def each(&block) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) + @array.each(&block) + + return if @overflow + return if @iterator.nil? + + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + if is_enabled && @from_gitaly + each_gitaly_patch(&block) + else + each_rugged_patch(&block) + end end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end def empty? @@ -72,23 +100,32 @@ module Gitlab end def over_safe_limits?(files) - files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes end - def each_patch - i = 0 - @array.each do |diff| - yield diff + def each_gitaly_patch + i = @array.length + + @iterator.each do |raw| + diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded) + + if raw.overflow_marker + @overflow = true + break + end + + yield @array[i] = diff i += 1 end + end - return if @overflow - return if @iterator.nil? + def each_rugged_patch + i = @array.length @iterator.each do |raw| @empty = false - if @enforce_limits && i >= @max_files + if @enforce_limits && i >= max_files @overflow = true break end @@ -104,7 +141,7 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -114,11 +151,6 @@ module Gitlab yield @array[i] = diff i += 1 end - - @populated = true - - # Allow iterator to be garbage-collected. It cannot be reused anyway. - @iterator = nil end end end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index f4e3b5e5129..4a43b9b444d 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class GitmodulesParser diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 5042916343b..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 666743006e5..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e51966313d4..f246393cfbc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,6 +45,8 @@ module Gitlab :bare?, to: :rugged + delegate :exists?, to: :gitaly_repository_client + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -56,17 +58,18 @@ module Gitlab end end - # Alias to old method for compatibility - def raw - rugged - end - def rugged - @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories) + @rugged ||= circuit_breaker.perform do + Rugged::Repository.new(path, alternates: alternate_object_directories) + end rescue Rugged::RepositoryError, Rugged::OSError raise NoRepository.new('no repository for such path') end + def circuit_breaker + @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -80,16 +83,14 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch + def branches + gitaly_migrate(:branches) do |is_enabled| + if is_enabled + gitaly_ref_client.branches + else + branches_filter end - end.compact - - sort_branches(branches, sort_by) + end end def reload_rugged @@ -107,7 +108,10 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) @@ -115,7 +119,7 @@ module Gitlab if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -162,20 +166,16 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end + gitaly_migrate(:tags) do |is_enabled| + if is_enabled + tags_from_gitaly + else + tags_from_rugged end - - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) - end.sort_by(&:name) + end end # Returns true if the given tag exists @@ -204,21 +204,10 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end - def repo_exists? - !!rugged - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -293,30 +282,15 @@ module Gitlab # Return repo size in megabytes def size - size = popen(%w(du -sk), path).first.strip.to_i - (size.to_f / 1024).round(2) - end - - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) + size = gitaly_migrate(:repository_size) do |is_enabled| + if is_enabled + size_by_gitaly + else + size_by_shelling_out + end end - greps + (size.to_f / 1024).round(2) end # Use the Rugged Walker API to build an array of commits. @@ -330,6 +304,7 @@ module Gitlab # after: Time.new(2016, 4, 21, 14, 32, 10) # ) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446 def log(options) default_options = { limit: 10, @@ -345,81 +320,18 @@ module Gitlab options = default_options.merge(options) options[:limit] ||= 0 options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end def count_commits(options) - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd += %W[--count #{options[:ref]}] - cmd += %W[-- #{options[:path]}] if options[:path].present? - - raw_output = IO.popen(cmd) { |io| io.read } - - raw_output.to_i + gitaly_migrate(:count_commits) do |is_enabled| + if is_enabled + count_commits_by_gitaly(options) + else + count_commits_by_shelling_out(options) + end + end end def sha_from_ref(ref) @@ -436,7 +348,9 @@ module Gitlab # Return a collection of Rugged::Commits between the two revspec arguments. # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for # a detailed list of valid arguments. - def commits_between(from, to) + # + # Gitaly note: JV: to be deprecated in favor of Commit.between + def rugged_commits_between(from, to) walker = Rugged::Walker.new(rugged) walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) @@ -454,7 +368,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -462,6 +376,13 @@ module Gitlab rugged.merge_base(from, to) end + # Gitaly note: JV: check gitlab-ee before removing this method. + def rugged_is_ancestor?(ancestor_id, descendant_id) + return false if ancestor_id.nil? || descendant_id.nil? + + merge_base_commit(ancestor_id, descendant_id) == ancestor_id + end + # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) gitaly_commit_client.is_ancestor(from, to) @@ -553,6 +474,7 @@ module Gitlab # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| if is_enabled @@ -567,6 +489,8 @@ module Gitlab end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) gitaly_migrate(:commit_count) do |is_enabled| if is_enabled @@ -581,20 +505,6 @@ module Gitlab end end - # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or - # tag name or a commit SHA. Valid +reset_type+ values are: - # - # [:soft] - # the head will be moved to the commit. - # [:mixed] - # will trigger a +:soft+ reset, plus the index will be replaced - # with the content of the commit tree. - # [:hard] - # will trigger a +:mixed+ reset and the working directory will be - # replaced with the content of the index. (Untracked and ignored files - # will be left alone) - delegate :reset, to: :rugged - # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # @@ -619,154 +529,6 @@ module Gitlab # TODO: implement this method end - # Check out the specified ref. Valid options are: - # - # :b - Create a new branch at +start_point+ and set HEAD to the new - # branch. - # - # * These options are passed to the Rugged::Repository#checkout method: - # - # :progress :: - # A callback that will be executed for checkout progress notifications. - # Up to 3 parameters are passed on each execution: - # - # - The path to the last updated file (or +nil+ on the very first - # invocation). - # - The number of completed checkout steps. - # - The number of total checkout steps to be performed. - # - # :notify :: - # A callback that will be executed for each checkout notification - # types specified with +:notify_flags+. Up to 5 parameters are passed - # on each execution: - # - # - An array containing the +:notify_flags+ that caused the callback - # execution. - # - The path of the current file. - # - A hash describing the baseline blob (or +nil+ if it does not - # exist). - # - A hash describing the target blob (or +nil+ if it does not exist). - # - A hash describing the workdir blob (or +nil+ if it does not - # exist). - # - # :strategy :: - # A single symbol or an array of symbols representing the strategies - # to use when performing the checkout. Possible values are: - # - # :none :: - # Perform a dry run (default). - # - # :safe :: - # Allow safe updates that cannot overwrite uncommitted data. - # - # :safe_create :: - # Allow safe updates plus creation of missing files. - # - # :force :: - # Allow all updates to force working directory to look like index. - # - # :allow_conflicts :: - # Allow checkout to make safe updates even if conflicts are found. - # - # :remove_untracked :: - # Remove untracked files not in index (that are not ignored). - # - # :remove_ignored :: - # Remove ignored files not in index. - # - # :update_only :: - # Only update existing files, don't create new ones. - # - # :dont_update_index :: - # Normally checkout updates index entries as it goes; this stops - # that. - # - # :no_refresh :: - # Don't refresh index/config/etc before doing checkout. - # - # :disable_pathspec_match :: - # Treat pathspec as simple list of exact match file paths. - # - # :skip_locked_directories :: - # Ignore directories in use, they will be left empty. - # - # :skip_unmerged :: - # Allow checkout to skip unmerged files (NOT IMPLEMENTED). - # - # :use_ours :: - # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). - # - # :use_theirs :: - # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). - # - # :update_submodules :: - # Recursively checkout submodules with same options (NOT - # IMPLEMENTED). - # - # :update_submodules_if_changed :: - # Recursively checkout submodules if HEAD moved in super repo (NOT - # IMPLEMENTED). - # - # :disable_filters :: - # If +true+, filters like CRLF line conversion will be disabled. - # - # :dir_mode :: - # Mode for newly created directories. Default: +0755+. - # - # :file_mode :: - # Mode for newly created files. Default: +0755+ or +0644+. - # - # :file_open_flags :: - # Mode for opening files. Default: - # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. - # - # :notify_flags :: - # A single symbol or an array of symbols representing the cases in - # which the +:notify+ callback should be invoked. Possible values are: - # - # :none :: - # Do not invoke the +:notify+ callback (default). - # - # :conflict :: - # Invoke the callback for conflicting paths. - # - # :dirty :: - # Invoke the callback for "dirty" files, i.e. those that do not need - # an update but no longer match the baseline. - # - # :updated :: - # Invoke the callback for any file that was changed. - # - # :untracked :: - # Invoke the callback for untracked files. - # - # :ignored :: - # Invoke the callback for ignored files. - # - # :all :: - # Invoke the callback for all these cases. - # - # :paths :: - # A glob string or an array of glob strings specifying which paths - # should be taken into account for the checkout operation. +nil+ will - # match all files. Default: +nil+. - # - # :baseline :: - # A Rugged::Tree that represents the current, expected contents of the - # workdir. Default: +HEAD+. - # - # :target_directory :: - # A path to an alternative workdir directory in which the checkout - # should be performed. - def checkout(ref, options = {}, start_point = "HEAD") - if options[:b] - rugged.branches.create(ref, start_point) - options.delete(:b) - end - default_options = { strategy: [:recreate_missing, :safe] } - rugged.checkout(ref, default_options.merge(options)) - end - # Delete the specified branch from the repository def delete_branch(branch_name) rugged.branches.delete(branch_name) @@ -779,7 +541,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -838,6 +601,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -864,6 +628,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -901,12 +666,142 @@ module Gitlab @attributes.attributes(path) end + def languages(ref = nil) + Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| + if is_enabled + gitaly_commit_client.languages(ref) + else + ref ||= rugged.head.target_id + languages = Linguist::Repository.new(rugged, ref).languages + total = languages.map(&:last).sum + + languages = languages.map do |language| + name, share = language + color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" + { + value: (share.to_f * 100 / total).round(2), + label: name, + color: color, + highlight: color + } + end + + languages.sort do |x, y| + y[:value] <=> x[:value] + end + end + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) + end + + def gitaly_repository_client + @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) + end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 @@ -930,7 +825,7 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT - gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Blob::MAX_DATA_DISPLAY_SIZE) + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) found_module = GitmodulesParser.new(gitmodules.data).parse[path] found_module && found_module['url'] @@ -974,46 +869,6 @@ module Gitlab submodule_data.select { |path, data| data['id'] } end - # Returns true if +commit+ introduced changes to +path+, using commit - # trees to make that determination. Uses the history simplification - # rules that `git log` uses by default, where a commit is omitted if it - # is TREESAME to any parent. - # - # If the +follow+ option is true and the file specified by +path+ was - # renamed, then the path value is set to the old path. - def commit_touches_path?(commit, path, follow, walker) - entry = tree_entry(commit, path) - - if commit.parents.empty? - # This is the root commit, return true if it has +path+ in its tree - return !entry.nil? - end - - num_treesame = 0 - commit.parents.each do |parent| - parent_entry = tree_entry(parent, path) - - # Only follow the first TREESAME parent for merge commits - if num_treesame > 0 - walker.hide(parent) - next - end - - if entry.nil? && parent_entry.nil? - num_treesame += 1 - elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] - num_treesame += 1 - end - end - - case num_treesame - when 0 - detect_rename(commit, commit.parents.first, path) if follow - true - else false - end - end - # Find the entry for +path+ in the tree for +commit+ def tree_entry(commit, path) pathname = Pathname.new(path) @@ -1041,110 +896,6 @@ module Gitlab tmp_entry end - # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was - # renamed in +commit+, then set +path+ to the old filename. - def detect_rename(commit, parent, path) - diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) - - # If +path+ is a filename, not a directory, then we should only have - # one delta. We don't need to follow renames for directories. - return nil if diff.each_delta.count > 1 - - delta = diff.each_delta.first - if delta.added? - full_diff = parent.diff(commit) - full_diff.find_similar! - - full_diff.each_delta do |full_delta| - if full_delta.renamed? && path == full_delta.new_file[:path] - # Look for the old path in ancestors - path.replace(full_delta.old_file[:path]) - end - end - end - end - - # Returns true if the index entry has the special file mode that denotes - # a submodule. - def submodule?(index_entry) - index_entry[:mode] == 57344 - end - - # Return a Rugged::Index that has read from the tree at +ref_name+ - def populated_index(ref_name) - commit = rev_parse_target(ref_name) - index = rugged.index - index.read_tree(commit.tree) - index - end - - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1173,20 +924,54 @@ module Gitlab end end - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) + def tags_from_rugged + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + end.sort_by(&:name) end - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + def last_commit_for_path_by_rugged(sha, path) + sha = last_commit_id_for_path(sha, path) + commit(sha) end - def gitaly_migrate(method, &block) - Gitlab::GitalyClient.migrate(method, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) + def tags_from_gitaly + gitaly_ref_client.tags + end + + def size_by_shelling_out + popen(%w(du -sk), path).first.strip.to_i + end + + def size_by_gitaly + gitaly_repository_client.repository_size + end + + def count_commits_by_gitaly(options) + gitaly_commit_client.commit_count(options[:ref], options) + end + + def count_commits_by_shelling_out(options) + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd += %W[--count #{options[:ref]}] + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + + raw_output.to_i end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb new file mode 100644 index 00000000000..e28be4b8a38 --- /dev/null +++ b/lib/gitlab/git/storage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Git + module Storage + class Inaccessible < StandardError + attr_reader :retry_after + + def initialize(message = nil, retry_after = nil) + super(message) + @retry_after = retry_after + end + end + + CircuitOpen = Class.new(Inaccessible) + + REDIS_KEY_PREFIX = 'storage_accessible:'.freeze + + def self.redis + Gitlab::Redis::SharedState + end + end + end +end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb new file mode 100644 index 00000000000..9ea9367d4b7 --- /dev/null +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -0,0 +1,144 @@ +module Gitlab + module Git + module Storage + class CircuitBreaker + FailureInfo = Struct.new(:last_failure, :failure_count) + + attr_reader :storage, + :hostname, + :storage_path, + :failure_count_threshold, + :failure_wait_time, + :failure_reset_time, + :storage_timeout + + delegate :last_failure, :failure_count, to: :failure_info + + def self.reset_all! + pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" + + Gitlab::Git::Storage.redis.with do |redis| + all_storage_keys = redis.keys(pattern) + redis.del(*all_storage_keys) unless all_storage_keys.empty? + end + + RequestStore.delete(:circuitbreaker_cache) + end + + def self.for_storage(storage) + cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do + Hash.new do |hash, storage_name| + hash[storage_name] = new(storage_name) + end + end + + cached_circuitbreakers[storage] + end + + def initialize(storage, hostname = Gitlab::Environment.hostname) + @storage = storage + @hostname = hostname + + config = Gitlab.config.repositories.storages[@storage] + @storage_path = config['path'] + @failure_count_threshold = config['failure_count_threshold'] + @failure_wait_time = config['failure_wait_time'] + @failure_reset_time = config['failure_reset_time'] + @storage_timeout = config['storage_timeout'] + end + + def perform + return yield unless Feature.enabled?('git_storage_circuit_breaker') + + check_storage_accessible! + + yield + end + + def circuit_broken? + return false if no_failures? + + recent_failure = last_failure > failure_wait_time.seconds.ago + too_many_failures = failure_count > failure_count_threshold + + recent_failure || too_many_failures + end + + # Memoizing the `storage_available` call means we only do it once per + # request when the storage is available. + # + # When the storage appears not available, and the memoized value is `false` + # we might want to try again. + def storage_available? + return @storage_available if @storage_available + + if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck + .storage_available?(storage_path, storage_timeout) + track_storage_accessible + else + track_storage_inaccessible + end + + @storage_available + end + + def check_storage_accessible! + if circuit_broken? + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + end + + unless storage_available? + raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time) + end + end + + def no_failures? + last_failure.blank? && failure_count == 0 + end + + def track_storage_inaccessible + @failure_info = FailureInfo.new(Time.now, failure_count + 1) + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hincrby(cache_key, :failure_count, 1) + redis.expire(cache_key, failure_reset_time) + end + end + end + + def track_storage_accessible + return if no_failures? + + @failure_info = FailureInfo.new(nil, 0) + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :last_failure, nil) + redis.hset(cache_key, :failure_count, 0) + end + end + end + + def failure_info + @failure_info ||= get_failure_info + end + + def get_failure_info + last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| + redis.hmget(cache_key, :last_failure, :failure_count) + end + + last_failure = Time.at(last_failure.to_i) if last_failure.present? + + FailureInfo.new(last_failure, failure_count.to_i) + end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end + end + end + end +end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb new file mode 100644 index 00000000000..91d8241f17b --- /dev/null +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -0,0 +1,55 @@ +module Gitlab + module Git + module Storage + module ForkedStorageCheck + extend self + + def storage_available?(path, timeout_seconds = 5) + status = timeout_check(path, timeout_seconds) + + status.success? + end + + def timeout_check(path, timeout_seconds) + filesystem_check_pid = check_filesystem_in_process(path) + + deadline = timeout_seconds.seconds.from_now.utc + wait_time = 0.01 + status = nil + + while status.nil? + if deadline > Time.now.utc + sleep(wait_time) + _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) + else + Process.kill('KILL', filesystem_check_pid) + # Blocking wait, so we are sure the process is gone before continuing + _pid, status = Process.wait2(filesystem_check_pid) + end + end + + status + end + + # This will spawn a new 2 processes to do the check: + # The outer child (waiter) will spawn another child process (stater). + # + # The stater is the process is performing the actual filesystem check + # the check might hang if the filesystem is acting up. + # In this case we will send a `KILL` to the waiter, which will still + # be responsive while the stater is hanging. + def check_filesystem_in_process(path) + spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null') + end + + def ruby_check + <<~RUBY_FILESYSTEM_CHECK + inner_pid = fork { File.stat(ARGV.first) } + Process.waitpid(inner_pid) + exit $?.exitstatus + RUBY_FILESYSTEM_CHECK + end + end + end + end +end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb new file mode 100644 index 00000000000..2d723147f4f --- /dev/null +++ b/lib/gitlab/git/storage/health.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Storage + class Health + attr_reader :storage_name, :info + + def self.pattern_for_storage(storage_name) + "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*" + end + + def self.for_all_storages + storage_names = Gitlab.config.repositories.storages.keys + results_per_storage = nil + + Gitlab::Git::Storage.redis.with do |redis| + keys_per_storage = all_keys_for_storages(storage_names, redis) + results_per_storage = load_for_keys(keys_per_storage, redis) + end + + results_per_storage.map do |name, info| + info.each { |i| i[:failure_count] = i[:failure_count].value.to_i } + new(name, info) + end + end + + def self.all_keys_for_storages(storage_names, redis) + keys_per_storage = {} + + redis.pipelined do + storage_names.each do |storage_name| + pattern = pattern_for_storage(storage_name) + + keys_per_storage[storage_name] = redis.keys(pattern) + end + end + + keys_per_storage + end + + def self.load_for_keys(keys_per_storage, redis) + info_for_keys = {} + + redis.pipelined do + keys_per_storage.each do |storage_name, keys_future| + info_for_storage = keys_future.value.map do |key| + { name: key, failure_count: redis.hget(key, :failure_count) } + end + + info_for_keys[storage_name] = info_for_storage + end + end + + info_for_keys + end + + def self.for_failing_storages + for_all_storages.select(&:failing?) + end + + def initialize(storage_name, info) + @storage_name = storage_name + @info = info + end + + def failing_info + @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 } + end + + def failing? + failing_info.any? + end + + def failing_on_hosts + @failing_on_hosts ||= failing_info.map do |info_for_host| + info_for_host[:name].split(':').last + end + end + + def failing_circuit_breakers + @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| + CircuitBreaker.new(storage_name, hostname) + end + end + + def total_failures + @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } + end + end + end + end +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b6d4e6cfe46..8e959c57c7c 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,36 +12,23 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) + Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| + if is_enabled + client = Gitlab::GitalyClient::CommitService.new(repository) + client.tree_entries(repository, sha, path) + else + tree_entries_from_rugged(repository, sha, path) + end end end + private + # Recursive search of tree id for path # # Ex. @@ -68,6 +57,34 @@ module Gitlab entry[:oid] end end + + def tree_entries_from_rugged(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode].to_s(8), + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha + ) + end + end end def initialize(options) diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 197a94487eb..c90ef282fdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -57,7 +57,7 @@ module Gitlab metadata = yield(metadata) if block_given? stub(service, storage).send(rpc, request, metadata) end - + def self.request_metadata(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) { metadata: { 'authorization' => "Bearer #{encoded_token}" } } @@ -86,8 +86,8 @@ module Gitlab feature.enabled? end - def self.migrate(feature) - is_enabled = feature_enabled?(feature) + def self.migrate(feature, status: MigrationStatus::OPT_IN) + is_enabled = feature_enabled?(feature, status: status) metric_name = feature.to_s metric_name += "_gitaly" if is_enabled diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob_service.rb index 0c398b46a08..7ea8e8d0857 100644 --- a/lib/gitlab/gitaly_client/blob.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,10 +1,10 @@ module Gitlab module GitalyClient - class Blob + class BlobService def initialize(repository) @gitaly_repo = repository.gitaly_repository end - + def get_blob(oid:, limit:) request = Gitaly::GetBlobRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index aafc0520664..61fe462d762 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -1,81 +1,13 @@ module Gitlab module GitalyClient class Commit - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + attr_reader :repository, :gitaly_commit - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @repository = repository - end - - def is_ancestor(ancestor_id, child_id) - request = Gitaly::CommitIsAncestorRequest.new( - repository: @gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) - - GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value - end - - def diff_from_parent(commit, options = {}) - request_params = commit_diff_request_params(commit, options) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request) - Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) - end - - def commit_deltas(commit) - request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request) - response.flat_map do |msg| - msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } - end - end - - def tree_entry(ref, path, limit = nil) - request = Gitaly::TreeEntryRequest.new( - repository: @gitaly_repo, - revision: ref, - path: path.dup.force_encoding(Encoding::ASCII_8BIT), - limit: limit.to_i - ) + delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit - response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request) - entry = response.first - return unless entry.oid.present? - - if entry.type == :BLOB - rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } - entry.data += rest_of_data - end - - entry - end - - def commit_count(ref) - request = Gitaly::CountCommitsRequest.new( - repository: @gitaly_repo, - revision: ref - ) - - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count - end - - private - - def commit_diff_request_params(commit, options = {}) - parent_id = commit.parents[0]&.id || EMPTY_TREE_ID - - { - repository: @gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - paths: options.fetch(:paths, []) - } + def initialize(repository, gitaly_commit) + @repository = repository + @gitaly_commit = gitaly_commit end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb new file mode 100644 index 00000000000..3f577ac8530 --- /dev/null +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -0,0 +1,193 @@ +module Gitlab + module GitalyClient + class CommitService + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + request_params[:enforce_limits] = options.fetch(:limits, true) + request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + + request = Gitaly::CommitDiffRequest.new(request_params) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true)) + end + + def commit_deltas(commit) + request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } + end + end + + def tree_entry(ref, path, limit = nil) + request = Gitaly::TreeEntryRequest.new( + repository: @gitaly_repo, + revision: ref, + path: path.dup.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + entry = response.first + return unless entry.oid.present? + + if entry.type == :BLOB + rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } + entry.data += rest_of_data + end + + entry + end + + def tree_entries(repository, revision, path) + request = Gitaly::GetTreeEntriesRequest.new( + repository: @gitaly_repo, + revision: revision, + path: path.presence || '.' + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + + response.flat_map do |message| + message.entries.map do |gitaly_tree_entry| + entry_path = gitaly_tree_entry.path.dup + Gitlab::Git::Tree.new( + id: gitaly_tree_entry.oid, + root_id: gitaly_tree_entry.root_oid, + type: gitaly_tree_entry.type.downcase, + mode: gitaly_tree_entry.mode.to_s(8), + name: File.basename(entry_path), + path: entry_path, + commit_id: gitaly_tree_entry.commit_oid + ) + end + end + end + + def commit_count(ref, options = {}) + request = Gitaly::CountCommitsRequest.new( + repository: @gitaly_repo, + revision: ref + ) + request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? + request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? + request.path = options[:path] if options[:path].present? + + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + end + + def last_commit_for_path(revision, path) + request = Gitaly::LastCommitForPathRequest.new( + repository: @gitaly_repo, + revision: revision.force_encoding(Encoding::ASCII_8BIT), + path: path.to_s.force_encoding(Encoding::ASCII_8BIT) + ) + + gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit + return unless gitaly_commit + + commit = GitalyClient::Commit.new(@repository, gitaly_commit) + Gitlab::Git::Commit.new(commit) + end + + def between(from, to) + request = Gitaly::CommitsBetweenRequest.new( + repository: @gitaly_repo, + from: from, + to: to + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + consume_commits_response(response) + end + + def find_all_commits(opts = {}) + request = Gitaly::FindAllCommitsRequest.new( + repository: @gitaly_repo, + revision: opts[:ref].to_s, + max_count: opts[:max_count].to_i, + skip: opts[:skip].to_i + ) + request.order = opts[:order].upcase if opts[:order].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + consume_commits_response(response) + end + + def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0) + request = Gitaly::CommitsByMessageRequest.new( + repository: @gitaly_repo, + query: query, + revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT), + path: path.to_s.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i, + offset: offset.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request) + consume_commits_response(response) + end + + def languages(ref = nil) + request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') + response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) + + response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } + end + + def raw_blame(revision, path) + request = Gitaly::RawBlameRequest.new( + repository: @gitaly_repo, + revision: revision, + path: path + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) + response.reduce("") { |memo, msg| memo << msg.data } + end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def consume_commits_response(response) + response.flat_map do |message| + message.commits.map do |gitaly_commit| + commit = GitalyClient::Commit.new(@repository, gitaly_commit) + Gitlab::Git::Commit.new(commit) + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 1e117b7e74a..d459c9a88fb 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,7 +1,7 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze attr_accessor(*FIELDS) diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notification_service.rb index 78ed433e6b8..326e6f7dafc 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Notifications + class NotificationService # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository @@ -10,7 +10,7 @@ module Gitlab def post_receive GitalyClient.call( @storage, - :notifications, + :notification_service, :post_receive, Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) ) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb deleted file mode 100644 index 6edc69de078..00000000000 --- a/lib/gitlab/gitaly_client/ref.rb +++ /dev/null @@ -1,82 +0,0 @@ -module Gitlab - module GitalyClient - class Ref - include Gitlab::EncodingHelper - - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @repository = repository - @gitaly_repo = repository.gitaly_repository - @storage = repository.storage - end - - def default_branch_name - request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request) - Gitlab::Git.branch_name(response.name) - end - - def branch_names - request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request) - consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } - end - - def tag_names - request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request) - consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } - end - - def find_ref_name(commit_id, ref_prefix) - request = Gitaly::FindRefNameRequest.new( - repository: @gitaly_repo, - commit_id: commit_id, - prefix: ref_prefix - ) - encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup) - end - - def count_tag_names - tag_names.count - end - - def count_branch_names - branch_names.count - end - - def local_branches(sort_by: nil) - request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) - request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref, :find_local_branches, request) - consume_branches_response(response) - end - - private - - def consume_refs_response(response) - response.flat_map { |message| message.names.map { |name| yield(name) } } - end - - def sort_by_param(sort_by) - sort_by = 'name' if sort_by == 'name_asc' - - enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) - raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value - enum_value - end - - def consume_branches_response(response) - response.flat_map do |message| - message.branches.map do |gitaly_branch| - Gitlab::Git::Branch.new( - @repository, - encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id - ) - end - end - end - end - end -end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb new file mode 100644 index 00000000000..b0f7548b7dc --- /dev/null +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -0,0 +1,148 @@ +module Gitlab + module GitalyClient + class RefService + include Gitlab::EncodingHelper + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def branches + request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + response.flat_map do |message| + message.branches.map do |branch| + gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) + target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) + Gitlab::Git.branch_name(response.name) + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) + consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) + consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } + end + + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @gitaly_repo, + commit_id: commit_id, + prefix: ref_prefix + ) + encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + end + + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) + consume_branches_response(response) + end + + def tags + request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + consume_tags_response(response) + end + + private + + def consume_refs_response(response) + response.flat_map { |message| message.names.map { |name| yield(name) } } + end + + def sort_by_param(sort_by) + sort_by = 'name' if sort_by == 'name_asc' + + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map do |message| + message.branches.map do |gitaly_branch| + Gitlab::Git::Branch.new( + @repository, + encode!(gitaly_branch.name.dup), + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) + ) + end + end + end + + def consume_tags_response(response) + response.flat_map do |message| + message.tags.map do |gitaly_tag| + if gitaly_tag.target_commit.present? + commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) + gitaly_commit = Gitlab::Git::Commit.new(commit) + end + + Gitlab::Git::Tag.new( + @repository, + encode!(gitaly_tag.name.dup), + gitaly_tag.id, + gitaly_commit, + encode!(gitaly_tag.message.chomp) + ) + end + end + end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name.dup, + author_email: response.commit_author.email.dup, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name.dup, + committer_email: response.commit_committer.email.dup + } + + Gitlab::Git::Commit.decorate(hash) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb new file mode 100644 index 00000000000..79ce784f2f2 --- /dev/null +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -0,0 +1,37 @@ +module Gitlab + module GitalyClient + class RepositoryService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def exists? + request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :repository_service, :exists, request).exists + end + + def garbage_collect(create_bitmap) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request) + end + + def repack_full(create_bitmap) + request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :repack_full, request) + end + + def repack_incremental + request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request) + end + + def repository_size + request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repository_size, request).size + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index a8c0b47e786..266b1a6fece 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -254,7 +254,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6c592ff469c..0396122eeb9 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -7,8 +7,8 @@ module Gitlab @project = project end - def path_with_namespace - "#{project.path_with_namespace}.wiki" + def disk_path + "#{project.disk_path}.wiki" end def import_url diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb new file mode 100644 index 00000000000..e1d1724295a --- /dev/null +++ b/lib/gitlab/gpg.rb @@ -0,0 +1,62 @@ +module Gitlab + module Gpg + extend self + + module CurrentKeyChain + extend self + + def add(key) + GPGME::Key.import(key) + end + + def fingerprints_from_key(key) + import = GPGME::Key.import(key) + + return [] if import.imported == 0 + + import.imports.map(&:fingerprint) + end + end + + def fingerprints_from_key(key) + using_tmp_keychain do + CurrentKeyChain.fingerprints_from_key(key) + end + end + + def primary_keyids_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).map { |raw_key| raw_key.primary_subkey.keyid } + end + end + + def user_infos_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| + raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + end + end + end + + def using_tmp_keychain + Dir.mktmpdir do |dir| + @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] + @original_dirs.push(dir) + + GPGME::Engine.home_dir = dir + + return_value = yield + + @original_dirs.pop + + GPGME::Engine.home_dir = @original_dirs[-1] + + return_value + end + end + end +end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb new file mode 100644 index 00000000000..55428b85207 --- /dev/null +++ b/lib/gitlab/gpg/commit.rb @@ -0,0 +1,85 @@ +module Gitlab + module Gpg + class Commit + attr_reader :commit + + def initialize(commit) + @commit = commit + + @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + end + + def has_signature? + !!(@signature_text && @signed_text) + end + + def signature + return unless has_signature? + + cached_signature = GpgSignature.find_by(commit_sha: commit.sha) + return cached_signature if cached_signature.present? + + using_keychain do |gpg_key| + create_cached_signature!(gpg_key) + end + end + + def update_signature!(cached_signature) + using_keychain do |gpg_key| + cached_signature.update_attributes!(attributes(gpg_key)) + end + end + + private + + def using_keychain + Gitlab::Gpg.using_tmp_keychain do + # first we need to get the keyid from the signature to query the gpg + # key belonging to the keyid. + # This way we can add the key to the temporary keychain and extract + # the proper signature. + gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + + if gpg_key + Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) + @verified_signature = nil + end + + yield gpg_key + end + end + + def verified_signature + @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + break verified_signature + end + end + + def create_cached_signature!(gpg_key) + GpgSignature.create!(attributes(gpg_key)) + end + + def attributes(gpg_key) + user_infos = user_infos(gpg_key) + + { + commit_sha: commit.sha, + project: commit.project, + gpg_key: gpg_key, + gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_user_name: user_infos[:name], + gpg_key_user_email: user_infos[:email], + valid_signature: gpg_signature_valid_signature_value(gpg_key) + } + end + + def gpg_signature_valid_signature_value(gpg_key) + !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + end + + def user_infos(gpg_key) + gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} + end + end + end +end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb new file mode 100644 index 00000000000..3bb491120ba --- /dev/null +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -0,0 +1,19 @@ +module Gitlab + module Gpg + class InvalidGpgSignatureUpdater + def initialize(gpg_key) + @gpg_key = gpg_key + end + + def run + GpgSignature + .select(:id, :commit_sha, :project_id) + .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) + .find_each do |gpg_signature| + Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7de6d4d9367..8b365dab185 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -27,10 +27,10 @@ module Gitlab Metric.new(name, value, labels) end - def with_timing(proc) + def with_timing start = Time.now - result = proc.call - yield result, Time.now.to_f - start.to_f + result = yield + [result, Time.now.to_f - start.to_f] end def catch_timeout(seconds, &block) diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 70da4080cae..eef97f54962 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,47 +10,48 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - tmp_file_path = tmp_file_path(storage_name) - - if !storage_stat_test(storage_name) + if !storage_circuitbreaker_test(storage_name) + HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) + elsif !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - elsif !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) else - HealthChecks::Result.new(true, nil, shard: storage_name) + with_temp_file(storage_name) do |tmp_file_path| + if !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + end end rescue RuntimeError => ex message = "unexpected error #{ex} when checking storage #{storage_name}" Rails.logger.error(message) HealthChecks::Result.new(false, message, shard: storage_name) - ensure - delete_test_file(tmp_file_path) end end end def metrics repository_storages.flat_map do |storage_name| - tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + storage_stat_metrics(storage_name), + storage_write_metrics(storage_name), + storage_read_metrics(storage_name), + storage_circuitbreaker_metrics(storage_name) ].flatten end end private - def operation_metrics(ok_metric, latency_metric, operation, **labels) - with_timing operation do |result, elapsed| - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - end + def operation_metrics(ok_metric, latency_metric, **labels) + result, elapsed = yield + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] rescue RuntimeError => ex Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] @@ -68,19 +69,36 @@ module Gitlab Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end - def tmp_file_path(storage_name) - Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + def with_temp_file(storage_name) + temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } + yield temp_file_path + ensure + delete_test_file(temp_file_path) end - def path(storage_name) + def storage_path(storage_name) storages_paths&.dig(storage_name, 'path') end + # All below test methods use shell commands to perform actions on storage volumes. + # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, + # we can rely on shell commands to be terminated once `timeout` kills them. + # + # However we also fallback to pure Ruby file operations in case a specific shell command is missing + # so we are still able to perform healthchecks and gather metrics from such system. + + def delete_test_file(tmp_path) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) + status.zero? + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + def storage_stat_test(storage_name) - stat_path = File.join(path(storage_name), '.') + stat_path = File.join(storage_path(storage_name), '.') begin _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status == 0 + status.zero? rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? end @@ -90,7 +108,7 @@ module Gitlab _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT written_bytes == RANDOM_STRING.length @@ -100,17 +118,47 @@ module Gitlab _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT file_contents = File.read(tmp_path) rescue Errno::ENOENT file_contents == RANDOM_STRING end - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status == 0 - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT + def storage_circuitbreaker_test(storage_name) + Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" } + rescue Gitlab::Git::Storage::Inaccessible + nil + end + + def storage_stat_metrics(storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do + with_timing { storage_stat_test(storage_name) } + end + end + + def storage_write_metrics(storage_name) + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + with_timing { storage_write_test(tmp_file_path) } + end + end + end + + def storage_read_metrics(storage_name) + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + storage_write_test(tmp_file_path) # writes data used by read test + with_timing { storage_read_test(tmp_file_path) } + end + end + end + + def storage_circuitbreaker_metrics(storage_name) + operation_metrics(:filesystem_circuitbreaker, + :filesystem_circuitbreaker_latency_seconds, + shard: storage_name) do + with_timing { storage_circuitbreaker_test(storage_name) } + end end end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..f5026171ba4 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -15,14 +15,13 @@ module Gitlab end def metrics - with_timing method(:check) do |result, elapsed| - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) - [ - metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) - ] - end + result, elapsed = with_timing(&method(:check)) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency_seconds", elapsed) + ] end private diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f3d489aad0d..5d106b5c075 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -7,13 +7,17 @@ module Gitlab 'es' => 'Español', 'de' => 'Deutsch', 'fr' => 'Français', - 'pt_BR' => 'Português(Brasil)', + 'pt_BR' => 'Português (Brasil)', 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文(香港)', - 'zh_TW' => '繁體中文(臺灣)', + 'zh_HK' => '繁體中文 (香港)', + 'zh_TW' => '繁體中文 (臺灣)', 'bg' => 'български', + 'ru' => 'Русский', 'eo' => 'Esperanto', - 'it' => 'Italiano' + 'it' => 'Italiano', + 'uk' => 'Українська', + 'ja' => '日本語', + 'ko' => '한국어' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index c8ad3a7a5e0..c5c05bfe2fb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -101,6 +101,7 @@ excluded_attributes: merge_requests: - :milestone_id - :ref_fetched + - :merge_jid award_emoji: - :awardable_id statuses: diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index c824d3ea9fc..32ca2809b2f 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle) + gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 62a2553675c..f9ae5079d7c 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,6 +24,7 @@ module Gitlab end def uploads_path + # TODO: decide what to do with uploads. We will use UUIDs here too? File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 52276cbcd9a..5404dc11a87 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,7 +8,7 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('github', 'GitHub', Github::Import), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index b75ae512d92..d9a79f7c291 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -1,55 +1,48 @@ module Gitlab class KeyFingerprint - include Gitlab::Popen + attr_reader :key, :ssh_key - attr_accessor :key + # Unqualified MD5 fingerprint for compatibility + delegate :fingerprint, to: :ssh_key, allow_nil: true def initialize(key) @key = key - end - - def fingerprint - cmd_status = 0 - cmd_output = '' - - Tempfile.open('gitlab_key_file') do |file| - file.puts key - file.rewind - - cmd = [] - cmd.push('ssh-keygen') - cmd.push('-E', 'md5') if explicit_fingerprint_algorithm? - cmd.push('-lf', file.path) - - cmd_output, cmd_status = popen(cmd, '/tmp') - end - - return nil unless cmd_status.zero? - # 16 hex bytes separated by ':', optionally starting with "MD5:" - fingerprint_matches = cmd_output.match(/(MD5:)?(?<fingerprint>(\h{2}:){15}\h{2})/) - return nil unless fingerprint_matches - - fingerprint_matches[:fingerprint] + @ssh_key = + begin + Net::SSH::KeyFactory.load_data_public_key(key) + rescue Net::SSH::Exception, NotImplementedError + end end - private - - def explicit_fingerprint_algorithm? - # OpenSSH 6.8 introduces a new default output format for fingerprints. - # Check the version and decide which command to use. - - version_output, version_status = popen(%w(ssh -V)) - return false unless version_status.zero? + def valid? + ssh_key.present? + end - version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) - return false unless version_matches + def type + return unless valid? - version_info = Gitlab::VersionInfo.new(version_matches[:major].to_i, version_matches[:minor].to_i) + parts = ssh_key.ssh_type.split('-') + parts.shift if parts[0] == 'ssh' - required_version_info = Gitlab::VersionInfo.new(6, 8) + parts[0].upcase + end - version_info >= required_version_info + def bits + return unless valid? + + case type + when 'RSA' + ssh_key.n.num_bits + when 'DSS', 'DSA' + ssh_key.p.num_bits + when 'ECDSA' + ssh_key.group.order.num_bits + when 'ED25519' + 256 + else + raise "Unsupported key type: #{type}" + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 7b05290e5cc..8867a91c244 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -101,7 +101,7 @@ module Gitlab end def user_attributes - %W(#{config.uid} cn mail dn) + %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] end end end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index 4745311402c..ed1de73f8c6 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -42,7 +42,7 @@ module Gitlab end def adapter - OmniAuth::LDAP::Adaptor.new(config.options.symbolize_keys) + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) end def config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 6fdf68641e2..c8f19cd52d5 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -2,6 +2,12 @@ module Gitlab module LDAP class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + attr_accessor :provider, :options def self.enabled? @@ -12,6 +18,12 @@ module Gitlab Gitlab.config.ldap.servers.values end + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + def self.providers servers.map { |server| server['provider_name'] } end @@ -39,7 +51,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption + encryption: encryption_options ) opts.merge!(auth_options) if has_auth? @@ -50,9 +62,10 @@ module Gitlab def omniauth_options opts = base_options.merge( base: base, - method: options['method'], + encryption: options['encryption'], filter: omniauth_user_filter, - name_proc: name_proc + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] ) if has_auth? @@ -62,6 +75,9 @@ module Gitlab ) end + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts end @@ -157,15 +173,37 @@ module Gitlab base_config.servers.values.find { |server| server['provider_name'] == provider } end - def encryption - case options['method'].to_s - when 'ssl' - :simple_tls - when 'tls' - :start_tls - else - nil - end + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts end def auth_options diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb index 219accfc029..716d20bb91a 100644 --- a/lib/gitlab/metrics/base_sampler.rb +++ b/lib/gitlab/metrics/base_sampler.rb @@ -1,20 +1,7 @@ require 'logger' module Gitlab module Metrics - class BaseSampler - def self.initialize_instance(*args) - raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) - at_exit(&@instance.method(:stop)) - @instance - end - - def self.instance - @instance - end - - attr_reader :running - + class BaseSampler < Daemon # interval - The sampling interval in seconds. def initialize(interval) interval_half = interval.to_f / 2 @@ -22,44 +9,7 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @mutex = Mutex.new - end - - def enabled? - true - end - - def start - return unless enabled? - - @mutex.synchronize do - return if running - @running = true - - @thread = Thread.new do - sleep(sleep_interval) - - while running - safe_sample - - sleep(sleep_interval) - end - end - end - end - - def stop - @mutex.synchronize do - return unless running - - @running = false - - if @thread - @thread.wakeup if @thread.alive? - @thread.join - @thread = nil - end - end + super() end def safe_sample @@ -81,7 +31,7 @@ module Gitlab # potentially missing anything that happens in between samples). # 2. Don't sample data at the same interval two times in a row. def sleep_interval - while step = @interval_steps.sample + while (step = @interval_steps.sample) if step != @last_step @last_step = step @@ -89,6 +39,25 @@ module Gitlab end end end + + private + + attr_reader :running + + def start_working + @running = true + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + + def stop_working + @running = false + end end end end diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb deleted file mode 100644 index b3da360be8f..00000000000 --- a/lib/gitlab/metrics/connection_rack_middleware.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module Metrics - class ConnectionRackMiddleware - def initialize(app) - @app = app - end - - def self.rack_request_count - @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count') - end - - def self.rack_response_count - @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count') - end - - def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count') - end - - def self.rack_execution_time - @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10]) - end - - def call(env) - method = env['REQUEST_METHOD'].downcase - started = Time.now.to_f - begin - ConnectionRackMiddleware.rack_request_count.increment(method: method) - - status, headers, body = @app.call(env) - - ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status) - [status, headers, body] - rescue - ConnectionRackMiddleware.rack_uncaught_errors_count.increment - raise - ensure - elapsed = Time.now.to_f - started - ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed) - end - end - end - end -end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index fb7bbc7cfc7..460dab47276 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -6,9 +6,11 @@ module Gitlab include Gitlab::CurrentSettings def metrics_folder_present? - ENV.has_key?('prometheus_multiproc_dir') && - ::Dir.exist?(ENV['prometheus_multiproc_dir']) && - ::File.writable?(ENV['prometheus_multiproc_dir']) + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) end def prometheus_metrics_enabled? diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb new file mode 100644 index 00000000000..5980a4ded2b --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -0,0 +1,39 @@ +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + class SidekiqMetricsExporter < Daemon + def enabled? + Gitlab::Metrics.metrics_folder_present? && settings.enabled + end + + def settings + Settings.monitoring.sidekiq_exporter + end + + private + + attr_reader :server + + def start_working + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + server.mount "/", Rack::Handler::WEBrick, rack_app + server.start + end + + def stop_working + server.shutdown + @server = nil + end + + def rack_app + Rack::Builder.app do + use Rack::Deflater + use ::Prometheus::Client::Rack::Exporter + run -> (env) { [404, {}, ['']] } + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b3f453e506d..e8330917e91 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -101,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end @@ -162,12 +166,17 @@ module Gitlab username ||= auth_hash.username email ||= auth_hash.email + valid_username = ::Namespace.clean_path(username) + + uniquify = Uniquify.new + valid_username = uniquify.string(valid_username) { |s| !DynamicPathValidator.valid_user_path?(s) } + name = auth_hash.name - name = ::Namespace.clean_path(username) if name.strip.empty? + name = valid_username if name.strip.empty? { name: name, - username: ::Namespace.clean_path(username), + username: valid_username, email: email, password: auth_hash.password, password_confirmation: auth_hash.password, diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index d81f825ef96..894bd5efae5 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -14,43 +14,42 @@ module Gitlab TOP_LEVEL_ROUTES = %w[ - .well-known + 404.html + 422.html + 500.html + 502.html + 503.html abuse_reports admin - all api + apple-touch-icon-precomposed.png + apple-touch-icon.png assets autocomplete ci dashboard + deploy.html explore + favicon.ico files groups health_check help - hooks import invites - issues jwt koding - member - merge_requests - new - notes notification_settings oauth profile projects public - repository robots.txt s search sent_notifications - services + slash-command-logo.png snippets - system - teams u unicorn_test unsubscribes diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 2da2ce45ebc..56112ec2301 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -2,7 +2,8 @@ module Gitlab module PerformanceBar include Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids'.freeze + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes def self.enabled?(user = nil) return false unless user && allowed_group_id @@ -15,7 +16,7 @@ module Gitlab end def self.allowed_user_ids - Rails.cache.fetch(ALLOWED_USER_IDS_KEY) do + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) if group diff --git a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb deleted file mode 100644 index d939a6ea18d..00000000000 --- a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This solves a bug with a X-Senfile header that wouldn't be set properly, see -# https://github.com/peek/peek-performance_bar/pull/27 -module Gitlab - module PerformanceBar - module PeekPerformanceBarWithRackBody - def call(env) - @env = env - reset_stats - - @total_requests += 1 - first_request if @total_requests == 1 - - env['process.request_start'] = @start.to_f - env['process.total_requests'] = total_requests - - status, headers, body = @app.call(env) - body = Rack::BodyProxy.new(body) { record_request } - [status, headers, body] - end - end - end -end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 574ae8731a5..67fee8c227d 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -1,4 +1,5 @@ # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb +# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb module Gitlab module PerformanceBar module PeekQueryTracker @@ -23,14 +24,20 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if RequestStore.active? && RequestStore.store[:peek_enabled] - track_query(data[:sql].strip, data[:binds], start, finish) + # data[:cached] is only available starting from Rails 5.1.0 + # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 + # Before that, data[:name] was set to 'CACHE' + # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end end end end def track_query(raw_query, bindings, start, finish) query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: '%.3f' % query.duration, sql: query.formatted_query } + query_info = { duration: query.duration.round(3), sql: query.formatted_query } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb new file mode 100644 index 00000000000..cf461adf697 --- /dev/null +++ b/lib/gitlab/project_template.rb @@ -0,0 +1,45 @@ +module Gitlab + class ProjectTemplate + attr_reader :title, :name + + def initialize(name, title) + @name, @title = name, title + end + + alias_method :logo, :name + + def file + archive_path.open + end + + def archive_path + Rails.root.join("vendor/project_templates/#{name}.tar.gz") + end + + def clone_url + "https://gitlab.com/gitlab-org/project-templates/#{name}.git" + end + + def ==(other) + name == other.name && title == other.title + end + + TEMPLATES_TABLE = [ + ProjectTemplate.new('rails', 'Ruby on Rails') + ].freeze + + class << self + def all + TEMPLATES_TABLE + end + + def find(name) + all.find { |template| template.name == name.to_s } + end + + def archive_directory + Rails.root.join("vendor_directory/project_templates") + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 67c69d9ccf3..69d055c901c 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -6,14 +6,13 @@ module Gitlab def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| - query_context = { - environment_slug: deployment.environment.slug, - environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, - timeframe_start: (deployment.created_at - 30.minutes).to_f, - timeframe_end: (deployment.created_at + 30.minutes).to_f - } - - query_metrics(query_context) + query_metrics( + common_query_context( + deployment.environment, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + ) + ) end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index b5a679ddd79..32fe8201a8d 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -5,15 +5,10 @@ module Gitlab include QueryAdditionalMetrics def query(environment_id) - Environment.find_by(id: environment_id).try do |environment| - query_context = { - environment_slug: environment.slug, - environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, - timeframe_start: 8.hours.ago.to_f, - timeframe_end: Time.now.to_f - } - - query_metrics(query_context) + ::Environment.find_by(id: environment_id).try do |environment| + query_metrics( + common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) + ) end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 66f29d95177..1d17d3cfd56 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class EnvironmentQuery < BaseQuery def query(environment_id) - Environment.find_by(id: environment_id).try do |environment| + ::Environment.find_by(id: environment_id).try do |environment| environment_slug = environment.slug timeframe_start = 8.hours.ago.to_f timeframe_end = Time.now.to_f diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index e44be770544..7ac6162b54d 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -42,15 +42,18 @@ module Gitlab end def process_query(context, query) - query_with_result = query.dup + query = query.dup result = if query.key?(:query_range) - client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + query[:query_range] %= context + client_query_range(query[:query_range], start: context[:timeframe_start], stop: context[:timeframe_end]) else - client_query(query[:query] % context, time: context[:timeframe_end]) + query[:query] %= context + client_query(query[:query], time: context[:timeframe_end]) end - query_with_result[:result] = result&.map(&:deep_symbolize_keys) - query_with_result + + query[:result] = result&.map(&:deep_symbolize_keys) + query end def available_metrics @@ -67,6 +70,16 @@ module Gitlab result.select { |group| group.metrics.any? } end + + def common_query_context(environment, timeframe_start:, timeframe_end:) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end, + ci_environment_slug: environment.slug, + kube_namespace: environment.project.kubernetes_service&.actual_namespace || '', + environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + } + end end end end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index a4a97236ffc..536765305e1 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -105,9 +105,32 @@ module Gitlab # # Awesome code block # end def command(*command_names, &block) + define_command(CommandDefinition, *command_names, &block) + end + + # Registers a new substitution which is recognizable from body of email or + # comment. + # It accepts aliases and takes a block with the formatted content. + # + # Example: + # + # command :my_substitution, :alias_for_my_substitution do |text| + # "#{text} MY AWESOME SUBSTITUTION" + # end + def substitution(*substitution_names, &block) + define_command(SubstitutionDefinition, *substitution_names, &block) + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] + end + + private + + def define_command(klass, *command_names, &block) name, *aliases = command_names - definition = CommandDefinition.new( + definition = klass.new( name, aliases: aliases, description: @description, @@ -130,10 +153,6 @@ module Gitlab @condition_block = nil @parse_params_block = nil end - - def definition_by_name(name) - command_definitions_by_name[name.to_sym] - end end end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 09576be7156..3ebfa3bd4b8 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -46,6 +46,8 @@ module Gitlab end end + content, commands = perform_substitutions(content, commands) + [content.strip, commands] end @@ -110,6 +112,26 @@ module Gitlab }mx end + def perform_substitutions(content, commands) + return unless content + + substitution_definitions = self.command_definitions.select do |definition| + definition.is_a?(Gitlab::QuickActions::SubstitutionDefinition) + end + + substitution_definitions.each do |substitution| + match_data = substitution.match(content) + if match_data + command = [substitution.name.to_s] + command << match_data[1] unless match_data[1].empty? + commands << command + end + content = substitution.perform_substitution(self, content) + end + + [content, commands] + end + def command_names(opts) command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb new file mode 100644 index 00000000000..032c49ed159 --- /dev/null +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -0,0 +1,24 @@ +module Gitlab + module QuickActions + class SubstitutionDefinition < CommandDefinition + # noop?=>true means these won't get extracted or removed by Gitlab::QuickActions::Extractor#extract_commands + # QuickActions::InterpretService#perform_substitutions handles them separately + def noop? + true + end + + def match(content) + content.match %r{^/#{all_names.join('|')} ?(.*)$} + end + + def perform_substitution(context, content) + return unless content + + all_names.each do |a_name| + content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + end + content + end + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 7668ecacc4b..f5b757ace77 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -33,7 +33,12 @@ module Gitlab def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue) + if project.issues_enabled? + @references[:all_issues] ||= references(:external_issue) + references(:issue) + else + @references[:external_issue] ||= references(:external_issue) + + references(:issue).select { |i| i.project_id != project.id } + end else @references[:issue] ||= references(:issue) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c1ee20b6977..1adc5ec952a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,17 +19,23 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def container_registry_reference_regex - Gitlab::PathRegex.git_reference_regex - end - ## - # Docker Distribution Registry 2.4.1 repository name rules + # Docker Distribution Registry repository / tag name rules + # + # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} end + ## + # We do not use regexp anchors here because these are not allowed when + # used as a routing constraint. + # + def container_registry_tag_regex + @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/ + end + def environment_name_regex_chars 'a-zA-Z0-9_/\\$\\{\\}\\. -' end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb new file mode 100644 index 00000000000..ccfe0d6bed3 --- /dev/null +++ b/lib/gitlab/request_forgery_protection.rb @@ -0,0 +1,39 @@ +# A module to check CSRF tokens in requests. +# It's used in API helpers and OmniAuth. +# Usage: GitLab::RequestForgeryProtection.call(env) + +module Gitlab + module RequestForgeryProtection + class Controller < ActionController::Base + protect_from_forgery with: :exception + + rescue_from ActionController::InvalidAuthenticityToken do |e| + logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" + logger.warn "Unlike the logs may suggest, this does not result in an actual 422 response to the user" + logger.warn "For API requests, the only effect is that `current_user` will be `nil` for the duration of the request" + + raise e + end + + def index + head :ok + end + end + + def self.app + @app ||= Controller.action(:index) + end + + def self.call(env) + app.call(env) + end + + def self.verified?(env) + call(env) + + true + rescue ActionController::InvalidAuthenticityToken + false + end + end +end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 877aa6e6a28..f3952657983 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -18,7 +18,11 @@ module Gitlab mapping = @map.find { |mapping| mapping[:source] === path } return unless mapping - path.sub(mapping[:source], mapping[:public]) + if mapping[:source].is_a?(String) + path.sub(mapping[:source], mapping[:public]) + else + mapping[:source].replace(path, mapping[:public]) + end end private @@ -35,7 +39,7 @@ module Gitlab source_pattern = source_pattern[1...-1].gsub('\/', '/') begin - source_pattern = /\A#{source_pattern}\z/ + source_pattern = Gitlab::UntrustedRegexp.new('\A' + source_pattern + '\z') rescue RegexpError => e raise FormatError, "Route map entry source is not a valid regular expression: #{e}" end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0baea092e6a..0cb28732402 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,3 +1,6 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab @@ -68,6 +71,7 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'add-project', storage, "#{name}.git"]) @@ -81,6 +85,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. @@ -99,12 +104,25 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # - def fetch_remote(storage, name, remote, forced: false, no_tags: false) + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced args << '--no-tags' if no_tags - gitlab_shell_fast_execute_raise_error(args) + vars = {} + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + end + end + + gitlab_shell_fast_execute_raise_error(args, vars) end # Move repository @@ -115,6 +133,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', storage, "#{path}.git", "#{new_path}.git"]) @@ -129,6 +148,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', forked_from_storage, "#{path}.git", forked_to_storage, @@ -143,6 +163,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'rm-project', storage, "#{name}.git"]) @@ -194,6 +215,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -207,6 +229,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -216,6 +239,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -241,6 +265,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end @@ -280,15 +305,15 @@ module Gitlab false end - def gitlab_shell_fast_execute_raise_error(cmd) - output, status = gitlab_shell_fast_execute_helper(cmd) + def gitlab_shell_fast_execute_raise_error(cmd, vars = {}) + output, status = gitlab_shell_fast_execute_helper(cmd, vars) raise Error, output unless status.zero? true end - def gitlab_shell_fast_execute_helper(cmd) - vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS) + def gitlab_shell_fast_execute_helper(cmd, vars = {}) + vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)) # Don't pass along the entire parent environment to prevent gitlab-shell # from wasting I/O by searching through GEM_PATH diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index e71eb15d604..93e00ab75a1 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -21,29 +21,34 @@ module Gitlab from = match[:from] to = match[:to] - actions = find_actions(from, to) + action = find_action(from, to) - if actions.none? - Gitlab::SlashCommands::Presenters::Deploy.new(nil).no_actions - elsif actions.one? - action = play!(from, to, actions.first) - Gitlab::SlashCommands::Presenters::Deploy.new(action).present(from, to) + if action.nil? + Gitlab::SlashCommands::Presenters::Deploy + .new(action).action_not_found else - Gitlab::SlashCommands::Presenters::Deploy.new(actions).too_many_actions + deployment = action.play(current_user) + + Gitlab::SlashCommands::Presenters::Deploy + .new(deployment).present(from, to) end end private - def play!(from, to, action) - action.play(current_user) - end - - def find_actions(from, to) + def find_action(from, to) environment = project.environments.find_by(name: from) - return [] unless environment + return unless environment - environment.actions_for(to).select(&:starts_environment?) + actions = environment.actions_for(to).select do |action| + action.starts_environment? + end + + if actions.many? + actions.find { |action| action.name == to.to_s } + else + actions.first + end end end end diff --git a/lib/gitlab/slash_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb index 87ea19b8806..3d96982b820 100644 --- a/lib/gitlab/slash_commands/issue_command.rb +++ b/lib/gitlab/slash_commands/issue_command.rb @@ -2,7 +2,7 @@ module Gitlab module SlashCommands class IssueCommand < BaseCommand def self.available?(project) - project.issues_enabled? && project.default_issues_tracker? + project.issues_enabled? end def collection diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb index b8dc77bd37b..ebae0f57f9b 100644 --- a/lib/gitlab/slash_commands/presenters/deploy.rb +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -3,17 +3,14 @@ module Gitlab module Presenters class Deploy < Presenters::Base def present(from, to) - message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + message = "Deployment started from #{from} to #{to}. " \ + "[Follow its progress](#{resource_url})." in_channel_response(text: message) end - def no_actions - ephemeral_response(text: "No action found to be executed") - end - - def too_many_actions - ephemeral_response(text: "Too many actions defined") + def action_not_found + ephemeral_response(text: "Couldn't find a deployment manual action.") end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb new file mode 100644 index 00000000000..7ce2e9d636e --- /dev/null +++ b/lib/gitlab/untrusted_regexp.rb @@ -0,0 +1,49 @@ +module Gitlab + # An untrusted regular expression is any regexp containing patterns sourced + # from user input. + # + # Ruby's built-in regular expression library allows patterns which complete in + # exponential time, permitting denial-of-service attacks. + # + # Not all regular expression features are available in untrusted regexes, and + # there is a strict limit on total execution time. See the RE2 documentation + # at https://github.com/google/re2/wiki/Syntax for more details. + class UntrustedRegexp + delegate :===, to: :regexp + + def initialize(pattern) + @regexp = RE2::Regexp.new(pattern, log_errors: false) + + raise RegexpError.new(regexp.error) unless regexp.ok? + end + + def replace_all(text, rewrite) + RE2.GlobalReplace(text, regexp, rewrite) + end + + def scan(text) + matches = scan_regexp.scan(text).to_a + matches.map!(&:first) if regexp.number_of_capturing_groups.zero? + matches + end + + def replace(text, rewrite) + RE2.Replace(text, regexp, rewrite) + end + + private + + attr_reader :regexp + + # RE2 scan operates differently to Ruby scan when there are no capture + # groups, so work around it + def scan_regexp + @scan_regexp ||= + if regexp.number_of_capturing_groups.zero? + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 35792d2d67f..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -52,15 +52,13 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) + snippet = object.noteable if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f19b325a126..748e0a29184 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -27,8 +27,8 @@ module Gitlab ci_pipeline_schedules: ::Ci::PipelineSchedule.count, deploy_keys: DeployKey.count, deployments: Deployment.count, - environments: Environment.count, - in_review_folder: Environment.in_review_folder.count, + environments: ::Environment.count, + in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, keys: Key.count, @@ -39,14 +39,14 @@ module Gitlab notes: Note.count, pages_domains: PagesDomain.count, projects: Project.count, - projects_prometheus_active: PrometheusService.active.count, + projects_imported_from_github: Project.where(import_type: 'github').count, protected_branches: ProtectedBranch.count, releases: Release.count, snippets: Snippet.count, todos: Todo.count, uploads: Upload.count, web_hooks: WebHook.count - } + }.merge(services_usage) } end @@ -63,6 +63,18 @@ module Gitlab usage_data end + + def services_usage + types = { + JiraService: :projects_jira_active, + SlackService: :projects_slack_notifications_active, + SlackSlashCommandsService: :projects_slack_slash_active, + PrometheusService: :projects_prometheus_active + } + + results = Service.unscoped.where(type: types.keys, active: true).group(:type).count + results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + end end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 3b922da7ced..d9a5af09f08 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,5 +1,11 @@ module Gitlab class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + attr_reader :user, :project def initialize(user, project: nil) @@ -28,43 +34,47 @@ module Gitlab true end - def can_create_tag?(ref) + request_cache def can_create_tag?(ref) return false unless can_access_git? - if ProtectedTag.protected?(project, ref) - project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + if protected?(ProtectedTag, project, ref) + protected_tag_accessible_to?(ref, action: :create) else user.can?(:push_code, project) end end - def can_delete_branch?(ref) + request_cache def can_delete_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) user.can?(:delete_protected_branch, project) else user.can?(:push_code, project) end end - def can_push_to_branch?(ref) + def can_update_branch?(ref) + can_push_to_branch?(ref) || can_merge_to_branch?(ref) + end + + request_cache def can_push_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) + protected_branch_accessible_to?(ref, action: :push) else user.can?(:push_code, project) end end - def can_merge_to_branch?(ref) + request_cache def can_merge_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) + if protected?(ProtectedBranch, project, ref) + protected_branch_accessible_to?(ref, action: :merge) else user.can?(:push_code, project) end @@ -81,5 +91,23 @@ module Gitlab def can_access_git? user && user.can?(:access_git) end + + def protected_branch_accessible_to?(ref, action:) + ProtectedBranch.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_branches) + end + + def protected_tag_accessible_to?(ref, action:) + ProtectedTag.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_tags) + end + + request_cache def protected?(kind, project, ref) + kind.protected?(project, ref) + end end end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 48f3d950779..c60bd91ea6e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -89,12 +89,12 @@ module Gitlab end def level_name(level) - level_name = 'Unknown' + level_name = N_('VisibilityLevel|Unknown') options.each do |name, lvl| level_name = name if lvl == level.to_i end - level_name + s_(level_name) end def level_value(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 4aef23b6aee..3f25e463412 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -35,7 +35,10 @@ module Gitlab when 'git_receive_pack' Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + Gitlab::GitalyClient.feature_enabled?( + :post_upload_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) when 'info_refs' true else @@ -62,10 +65,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -176,7 +190,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -192,6 +206,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb new file mode 100644 index 00000000000..05668c69006 --- /dev/null +++ b/lib/haml_lint/inline_javascript.rb @@ -0,0 +1,16 @@ +unless Rails.env.production? + require 'haml_lint/haml_visitor' + require 'haml_lint/linter' + require 'haml_lint/linter_registry' + + module HamlLint + class Linter::InlineJavaScript < Linter + include LinterRegistry + + def visit_filter(node) + return unless node.filter_type == 'javascript' + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') + end + end + end +end diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index 3d60618006c..d80cd7d2a4e 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -24,6 +24,10 @@ module Mattermost json_response session.post(path, options) end + def delete(session, path, options) + json_response session.delete(path, options) + end + def session_get(path, options = {}) with_session do |session| get(session, path, options) @@ -36,6 +40,12 @@ module Mattermost end end + def session_delete(path, options = {}) + with_session do |session| + delete(session, path, options) + end + end + def json_response(response) json_response = JSON.parse(response.body) diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 688a79c0441..ef08bd46e17 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -36,11 +36,12 @@ module Mattermost def with_session with_lease do - raise Mattermost::NoSessionError unless create + create begin yield self - rescue Errno::ECONNREFUSED + rescue Errno::ECONNREFUSED => e + Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) raise Mattermost::NoSessionError ensure destroy @@ -85,10 +86,12 @@ module Mattermost private def create - return unless oauth_uri - return unless token_uri + raise Mattermost::NoSessionError unless oauth_uri + raise Mattermost::NoSessionError unless token_uri @token = request_token + raise Mattermost::NoSessionError unless @token + @headers = { Authorization: "Bearer #{@token}" } @@ -106,11 +109,16 @@ module Mattermost @oauth_uri = nil response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) - return unless 300 <= response.code && response.code < 400 + return unless (300...400) === response.code redirect_uri = response.headers['location'] return unless redirect_uri + oauth_cookie = parse_cookie(response) + @headers = { + Cookie: oauth_cookie.to_cookie_string + } + @oauth_uri = URI.parse(redirect_uri) end @@ -124,7 +132,7 @@ module Mattermost def request_token response = get(token_uri, follow_redirects: false) - if 200 <= response.code && response.code < 400 + if (200...400) === response.code response.headers['token'] end end @@ -156,5 +164,11 @@ module Mattermost rescue Errno::ECONNREFUSED => e raise Mattermost::ConnectionError.new(e.message) end + + def parse_cookie(response) + cookie_hash = CookieHash.new + response.get_fields('Set-Cookie').each { |c| cookie_hash.add_cookies(c) } + cookie_hash + end end end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 2cdbbdece16..b2511f3af1d 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -14,5 +14,12 @@ module Mattermost type: type }.to_json) end + + # The deletion is done async, so the response is fast. + # On the mattermost side, this triggers an soft deletion first, after which + # the actuall data is removed + def destroy(team_id:) + session_delete("/api/v4/teams/#{team_id}?permanent=true") + end end end diff --git a/lib/omni_auth/request_forgery_protection.rb b/lib/omni_auth/request_forgery_protection.rb deleted file mode 100644 index 69155131d8d..00000000000 --- a/lib/omni_auth/request_forgery_protection.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Protects OmniAuth request phase against CSRF. - -module OmniAuth - module RequestForgeryProtection - class Controller < ActionController::Base - protect_from_forgery with: :exception - - def index - head :ok - end - end - - def self.app - @app ||= Controller.action(:index) - end - - def self.call(env) - app.call(env) - end - end -end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f93336346..2f2de083dc0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -291,7 +291,7 @@ start_gitlab() { fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The GitLab Workhorse is already running with pid $spid, not restarting" + echo "The GitLab Workhorse is already running with pid $hpid, not restarting" else # No need to remove a socket, gitlab-workhorse does this itself. # Because gitlab-workhorse has multiple executables we need to fix @@ -313,7 +313,7 @@ start_gitlab() { if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages is already running with pid $spid, not restarting" + echo "The GitLab Pages is already running with pid $gppid, not restarting" else $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ @@ -421,7 +421,7 @@ print_status() { fi if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages with pid $mpid is running." + echo "The GitLab Pages with pid $gppid is running." else printf "The GitLab Pages is \033[31mnot running\033[0m.\n" fi diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d6..564aa141952 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -5,12 +5,12 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| cursor = REDIS_SCAN_START_STOP loop do cursor, keys = redis.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: REDIS_CLEAR_BATCH_SIZE ) diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b27f7475115..b48e4dce445 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -5,7 +5,7 @@ namespace :gettext do # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate folders = %W(app lib config #{locale_path}).join(',') - exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( "{#{folders}}/**/*.{#{exts}}" diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 003d57adbbd..259a755d724 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -4,6 +4,7 @@ namespace :gitlab do task compile: [ 'yarn:check', 'rake:assets:precompile', + 'gettext:po_to_json', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 858f1cd7b34..dbb3b827b9a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -527,7 +527,7 @@ namespace :gitlab do repo_dirs = user.authorized_projects.map do |p| File.join( p.repository_storage_path, - "#{p.path_with_namespace}.git" + "#{p.disk_path}.git" ) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index a8db5701d0b..aaf00bd703a 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -19,7 +19,10 @@ namespace :gitlab do Dir.chdir(args.dir) do create_gitaly_configuration - run_command!([command]) + # In CI we run scripts/gitaly-test-build instead of this command + unless ENV['CI'].present? + Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT] + [command]) } + end end end @@ -30,7 +33,9 @@ namespace :gitlab do puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" puts "# This is in TOML format suitable for use in Gitaly's config.toml file." - puts gitaly_configuration_toml + # Exclude gitaly-ruby configuration because that depends on the gitaly + # installation directory. + puts gitaly_configuration_toml(gitaly_ruby: false) end private @@ -41,7 +46,7 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. - def gitaly_configuration_toml + def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -60,6 +65,8 @@ namespace :gitlab do end config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } TOML.dump(config) end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index ffcc76e5498..b732db9db6e 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(project.repository_storage_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.disk_path) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ee2cdcdea1b..42825f29e32 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -80,7 +80,7 @@ namespace :gitlab do print '-' else if Gitlab::Shell.new.add_repository(project.repository_storage_path, - project.path_with_namespace) + project.disk_path) print '.' else print 'F' diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 964aa0fe1bc..28b2d86eed2 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -153,7 +153,6 @@ module Gitlab clone_repo(repo, target_dir) unless Dir.exist?(target_dir) checkout_version(version, target_dir) - reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) @@ -161,12 +160,8 @@ module Gitlab end def checkout_version(version, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) - end - - def reset_to_version(version, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet origin #{version}]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout -f --quiet FETCH_HEAD --]) end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 59c32bbe7a4..a7e30423c7a 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -4,6 +4,55 @@ namespace :gitlab do TEMPLATE_DATA.each { |template| update(template) } end + desc "GitLab | Update project templates" + task :update_project_templates do + if Rails.env.production? + puts "This rake task is not meant fo production instances".red + exit(1) + end + admin = User.find_by(admin: true) + + unless admin + puts "No admin user could be found".red + exit(1) + end + + Gitlab::ProjectTemplate.all.each do |template| + params = { + import_url: template.clone_url, + namespace_id: admin.namespace.id, + path: template.title, + skip_wiki: true + } + + puts "Creating project for #{template.name}" + project = Projects::CreateService.new(admin, params).execute + + loop do + if project.finished? + puts "Import finished for #{template.name}" + break + end + + if project.failed? + puts "Failed to import from #{project_params[:import_url]}".red + exit(1) + end + + puts "Waiting for the import to finish" + + sleep(5) + project.reload + end + + Projects::ImportExport::ExportService.new(project, admin).execute + FileUtils.cp(project.export_project_path, template.archive_path) + Projects::DestroyService.new(admin, project).execute + puts "Exported #{template.name}".green + end + puts "Done".green + end + def update(template) sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index 609dfaa48e3..ad2d034b0b4 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -1,5 +1,6 @@ unless Rails.env.production? require 'haml_lint/rake_task' + require 'haml_lint/inline_javascript' HamlLint::RakeTask.new end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 50b8e331469..96b8f59242c 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,7 +7,7 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { url: 'https://api.github.com', token: token, verbose: true } + @options = { token: token, verbose: true } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first @@ -62,6 +62,7 @@ class GithubImport visibility_level: visibility_level, import_type: 'github', import_source: @repo['full_name'], + import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"), skip_wiki: @repo['has_wiki'] ).execute end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 4108cee08b4..9cc986535e1 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') +require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' @@ -12,5 +13,6 @@ task setup_postgresql: :environment do AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up IndexRoutesPathForLike.new.up + AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up end |