diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2018-06-27 16:15:06 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2018-06-27 16:15:06 +0800 |
| commit | 849f9995d97c85d88b36a40ee563f7dd51fdc3f1 (patch) | |
| tree | 3523089d253b001f7e3b029266399e497407441b /lib | |
| parent | ef6b3e0271d226462bed5f899f3964cf5652978c (diff) | |
| parent | 87f7597a4fb7852fc81f830158cdfd5fdec8fac4 (diff) | |
| download | gitlab-ce-849f9995d97c85d88b36a40ee563f7dd51fdc3f1.tar.gz | |
Merge remote-tracking branch 'upstream/master' into 14995-custom_wiki_sidebar
* upstream/master: (4180 commits)
Enable frozen string literals for app/workers/*.rb
Resolve "Search dropdown hides & shows when typing"
Revert merge request widget button max height
Update CHANGELOG.md for 11.0.2
Update external link icon in header user dropdown
Added Diff Viewer to new VUE based MR page
Fixed eslint failure in IDE spec helpers
Use refs instead of querySelector.
Show file in tree on WebIDE open
Resolve "Remove unused bootstrap component CSS"
Resolve "Explain what Groups are in the New Group page"
[QA] Make sure we wait for the deploy key list to load
Update _scopes_form.html.haml to remove duplicate information
Use the branch instead of the tag to install
port the EE changes
Add index on deployable_type/id for deployments
Add a helper to rename a column using a background migration
Fix performance bottleneck when rendering large wiki pages
Port Namespace#root_ancestor to CE
Remove duplicate spec
...
Diffstat (limited to 'lib')
468 files changed, 8392 insertions, 8591 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 62ffebeacb0..e2ad3c5f4e3 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -8,60 +8,28 @@ module API PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze - use GrapeLogging::Middleware::RequestLogger, - logger: Logger.new(LOG_FILENAME), - formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, - include: [ - GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new, - Gitlab::GrapeLogging::Loggers::UserLogger.new - ] + insert_before Grape::Middleware::Error, + GrapeLogging::Middleware::RequestLogger, + logger: Logger.new(LOG_FILENAME), + formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, + include: [ + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::UserLogger.new, + Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new + ] allow_access_with_scope :api prefix :api - version %w(v3 v4), using: :path - version 'v3', using: :path do - helpers ::API::V3::Helpers - helpers ::API::Helpers::CommonHelpers - - mount ::API::V3::AwardEmoji - mount ::API::V3::Boards - mount ::API::V3::Branches - mount ::API::V3::BroadcastMessages - mount ::API::V3::Builds - mount ::API::V3::Commits - mount ::API::V3::DeployKeys - mount ::API::V3::Environments - mount ::API::V3::Files - mount ::API::V3::Groups - mount ::API::V3::Issues - mount ::API::V3::Labels - mount ::API::V3::Members - mount ::API::V3::MergeRequestDiffs - mount ::API::V3::MergeRequests - mount ::API::V3::Notes - mount ::API::V3::Pipelines - mount ::API::V3::ProjectHooks - mount ::API::V3::Milestones - mount ::API::V3::Projects - mount ::API::V3::ProjectSnippets - mount ::API::V3::Repositories - mount ::API::V3::Runners - mount ::API::V3::Services - mount ::API::V3::Settings - mount ::API::V3::Snippets - mount ::API::V3::Subscriptions - mount ::API::V3::SystemHooks - mount ::API::V3::Tags - mount ::API::V3::Templates - mount ::API::V3::Todos - mount ::API::V3::Triggers - mount ::API::V3::Users - mount ::API::V3::Variables + route :any, '*path' do + error!('API V3 is no longer supported. Use API V4 instead.', 410) + end end + version 'v4', using: :path + before do header['X-Frame-Options'] = 'SAMEORIGIN' header['X-Content-Type-Options'] = 'nosniff' @@ -78,6 +46,14 @@ module API rack_response({ 'message' => '404 Not found' }.to_json, 404) end + rescue_from UploadedFile::InvalidPathError do |e| + rack_response({ 'message' => e.message }.to_json, 400) + end + + rescue_from ObjectStorage::RemoteStoreError do |e| + rack_response({ 'message' => e.message }.to_json, 500) + end + # Retain 405 error rather than a 500 error for Grape 0.15.0+. # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes rescue_from Grape::Exceptions::MethodNotAllowed do |e| @@ -107,6 +83,7 @@ module API # Keep in alphabetical order mount ::API::AccessRequests mount ::API::Applications + mount ::API::Avatar mount ::API::AwardEmoji mount ::API::Badges mount ::API::Boards @@ -131,6 +108,7 @@ module API mount ::API::Keys mount ::API::Labels mount ::API::Lint + mount ::API::Markdown mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests @@ -146,6 +124,7 @@ module API mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones + mount ::API::ProjectSnapshots mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c2113551207..c17089759de 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -45,7 +45,9 @@ module API user = find_user_from_sources return unless user - forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + unless api_access_allowed?(user) + forbidden!(api_access_denied_message(user)) + end user end @@ -72,6 +74,14 @@ module API end end end + + def api_access_allowed?(user) + Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + end + + def api_access_denied_message(user) + Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message + end end module ClassMethods diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb new file mode 100644 index 00000000000..70219bc8ea0 --- /dev/null +++ b/lib/api/avatar.rb @@ -0,0 +1,21 @@ +module API + class Avatar < Grape::API + resource :avatar do + desc 'Return avatar url for a user' do + success Entities::Avatar + end + params do + requires :email, type: String, desc: 'Public email address of the user' + optional :size, type: Integer, desc: 'Single pixel dimension for Gravatar images' + end + get do + forbidden!('Unauthorized access') unless can?(current_user, :read_users_list) + + user = User.find_by_public_email(params[:email]) + user ||= User.new(email: params[:email]) + + present user, with: Entities::Avatar, size: params[:size] + end + end + end +end diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 334948b2995..8ceffe9c5ef 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -127,6 +127,7 @@ module API end destroy_conditionally!(badge) + body false end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 13cfba728fa..4b223a391ae 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -45,6 +45,7 @@ module API present( paginate(::Kaminari.paginate_array(branches)), with: Entities::Branch, + current_user: current_user, project: user_project, merged_branch_names: merged_branch_names ) @@ -63,7 +64,7 @@ module API get do branch = find_branch!(params[:branch]) - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end end @@ -101,7 +102,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -121,7 +122,7 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end desc 'Create branch' do @@ -140,6 +141,7 @@ module API if result[:status] == :success present result[:branch], with: Entities::Branch, + current_user: current_user, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 982f45425a3..964780cba6a 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -15,19 +15,21 @@ module API end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' - optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' - optional :path, type: String, desc: 'The file path' - optional :all, type: Boolean, desc: 'Every commit will be returned' + optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' + optional :path, type: String, desc: 'The file path' + optional :all, type: Boolean, desc: 'Every commit will be returned' + optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response' use :pagination end get ':id/repository/commits' do - path = params[:path] + path = params[:path] before = params[:until] - after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] + after = params[:since] + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] - all = params[:all] + all = params[:all] + with_stats = params[:with_stats] commits = user_project.repository.commits(ref, path: path, @@ -47,7 +49,9 @@ module API paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present paginate(paginated_commits), with: Entities::Commit + serializer = with_stats ? Entities::CommitWithStats : Entities::Commit + + present paginate(paginated_commits), with: serializer end desc 'Commit multiple file changes as one commit' do @@ -231,6 +235,20 @@ module API render_api_error!("Failed to save note #{note.errors.messages}", 400) end end + + desc 'Get Merge Requests associated with a commit' do + success Entities::MergeRequestBasic + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to find Merge Requests' + use :pagination + end + get ':id/repository/commits/:sha/merge_requests', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + present paginate(commit.merge_requests), with: Entities::MergeRequestBasic + end end end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b0b7b50998f..b7aadc27e71 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -54,7 +54,7 @@ module API present key, with: Entities::DeployKeysProject end - desc 'Add new deploy key to currently authenticated user' do + desc 'Add new deploy key to a project' do success Entities::DeployKeysProject end params do @@ -66,33 +66,32 @@ module API params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys_projects + deploy_key_project = user_project.deploy_keys_projects .joins(:deploy_key) .find_by(keys: { key: params[:key] }) - if key - present key, with: Entities::DeployKeysProject + if deploy_key_project + present deploy_key_project, with: Entities::DeployKeysProject break end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key - added_key = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) + deploy_key_project = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) - present added_key, with: Entities::DeployKeysProject + present deploy_key_project, with: Entities::DeployKeysProject break end # Create a new deploy key - key_attributes = { can_push: !!params[:can_push], - deploy_key_attributes: declared_params.except(:can_push) } - key = add_deploy_keys_project(user_project, key_attributes) + deploy_key_attributes = declared_params.except(:can_push).merge(user: current_user) + deploy_key_project = add_deploy_keys_project(user_project, deploy_key_attributes: deploy_key_attributes, can_push: !!params[:can_push]) - if key.valid? - present key, with: Entities::DeployKeysProject + if deploy_key_project.valid? + present deploy_key_project, with: Entities::DeployKeysProject else - render_validation_error!(key) + render_validation_error!(deploy_key_project) end end @@ -149,10 +148,10 @@ module API requires :key_id, type: Integer, desc: 'The ID of the deploy key' end delete ":id/deploy_keys/:key_id" do - key = user_project.deploy_keys.find(params[:key_id]) - not_found!('Deploy Key') unless key + deploy_key_project = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + not_found!('Deploy Key') unless deploy_key_project - destroy_conditionally!(key) + destroy_conditionally!(deploy_key_project) end end end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 6abd575b6ad..13c34e3473a 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -5,11 +5,12 @@ module API before { authenticate! } - NOTEABLE_TYPES = [Issue, Snippet].freeze + NOTEABLE_TYPES = [Issue, Snippet, MergeRequest, Commit].freeze NOTEABLE_TYPES.each do |noteable_type| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize + noteables_path = noteable_type == Commit ? "repository/#{noteables_str}" : noteables_str params do requires :id, type: String, desc: "The ID of a #{parent_type}" @@ -19,14 +20,12 @@ module API success Entities::Discussion end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end - get ":id/#{noteables_str}/:noteable_id/discussions" do + get ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - return not_found!("Discussions") unless can?(current_user, noteable_read_ability_name(noteable), noteable) - notes = noteable.notes .inc_relations_for_view .includes(:noteable) @@ -43,14 +42,14 @@ module API end params do requires :discussion_id, type: String, desc: 'The ID of a discussion' - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end - get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) - if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) - return not_found!("Discussion") + if notes.empty? + break not_found!("Discussion") end discussion = Discussion.build(notes, noteable) @@ -62,19 +61,36 @@ module API success Entities::Discussion end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' requires :body, type: String, desc: 'The content of a note' optional :created_at, type: String, desc: 'The creation date of the note' + optional :position, type: Hash do + requires :base_sha, type: String, desc: 'Base commit SHA in the source branch' + requires :start_sha, type: String, desc: 'SHA referencing commit in target branch' + requires :head_sha, type: String, desc: 'SHA referencing HEAD of this merge request' + requires :position_type, type: String, desc: 'Type of the position reference', values: %w(text image) + optional :new_path, type: String, desc: 'File path after change' + optional :new_line, type: Integer, desc: 'Line number after change' + optional :old_path, type: String, desc: 'File path before change' + optional :old_line, type: Integer, desc: 'Line number before change' + optional :width, type: Integer, desc: 'Width of the image' + optional :height, type: Integer, desc: 'Height of the image' + optional :x, type: Integer, desc: 'X coordinate in the image' + optional :y, type: Integer, desc: 'Y coordinate in the image' + end end - post ":id/#{noteables_str}/:noteable_id/discussions" do + post ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + type = params[:position] ? 'DiffNote' : 'DiscussionNote' + id_key = noteable.is_a?(Commit) ? :commit_id : :noteable_id opts = { note: params[:body], created_at: params[:created_at], - type: 'DiscussionNote', + type: type, noteable_type: noteables_str.classify, - noteable_id: noteable.id + position: params[:position], + id_key => noteable.id } note = create_note(noteable, opts) @@ -91,14 +107,14 @@ module API end params do requires :discussion_id, type: String, desc: 'The ID of a discussion' - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end - get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) - if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) - return not_found!("Notes") + if notes.empty? + break not_found!("Notes") end present notes, with: Entities::Note @@ -108,17 +124,17 @@ module API success Entities::Note end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :body, type: String, desc: 'The content of a note' optional :created_at, type: String, desc: 'The creation date of the note' end - post ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do + post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) - return not_found!("Discussion") if notes.empty? - return bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? + break not_found!("Discussion") if notes.empty? + break bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? opts = { note: params[:body], @@ -139,11 +155,11 @@ module API success Entities::Note end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end - get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) get_note(noteable, params[:note_id]) @@ -153,30 +169,52 @@ module API success Entities::Note end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' - requires :body, type: String, desc: 'The content of a note' + optional :body, type: String, desc: 'The content of a note' + optional :resolved, type: Boolean, desc: 'Mark note resolved/unresolved' + exactly_one_of :body, :resolved end - put ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - update_note(noteable, params[:note_id]) + if params[:resolved].nil? + update_note(noteable, params[:note_id]) + else + resolve_note(noteable, params[:note_id], params[:resolved]) + end end desc "Delete a comment in a #{noteable_type.to_s.downcase} discussion" do success Entities::Note end params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end - delete ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) delete_note(noteable, params[:note_id]) end + + if Noteable::RESOLVABLE_TYPES.include?(noteable_type.to_s) + desc "Resolve/unresolve an existing #{noteable_type.to_s.downcase} discussion" do + success Entities::Discussion + end + params do + requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :discussion_id, type: String, desc: 'The ID of a discussion' + requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved' + end + put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do + noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + + resolve_discussion(noteable, params[:discussion_id], params[:resolved]) + end + end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 16147ee90c9..bb48a86fe9e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -72,7 +72,7 @@ module API class ProjectHook < Hook expose :project_id, :issues_events, :confidential_issues_events - expose :note_events, :pipeline_events, :wiki_page_events + expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events end @@ -125,7 +125,7 @@ module API # (fixed in https://github.com/rails/rails/pull/25976). project.tags.map(&:name).sort end - expose :ssh_url_to_repo, :http_url_to_repo, :web_url + expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url expose :avatar_url do |project, options| project.avatar_url(only_path: false) end @@ -136,6 +136,7 @@ module API def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) + .preload(:import_state) .preload(namespace: [:route, :owner], tags: :taggings) end @@ -149,11 +150,11 @@ module API expose_url(api_v4_projects_path(id: project.id)) end - expose :issues, if: -> (*args) { issues_available?(*args) } do |project| + expose :issues, if: -> (project, options) { issues_available?(project, options) } do |project| expose_url(api_v4_projects_issues_path(id: project.id)) end - expose :merge_requests, if: -> (*args) { mrs_available?(*args) } do |project| + expose :merge_requests, if: -> (project, options) { mrs_available?(project, options) } do |project| expose_url(api_v4_projects_merge_requests_path(id: project.id)) end @@ -206,6 +207,7 @@ module API expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved expose :printing_merge_request_link_enabled + expose :merge_method expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics @@ -241,13 +243,18 @@ module API expose :requested_at end - class Group < Grape::Entity - expose :id, :name, :path, :description, :visibility + class BasicGroupDetails < Grape::Entity + expose :id + expose :web_url + expose :name + end + + class Group < BasicGroupDetails + expose :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url do |group, options| group.avatar_url(only_path: false) end - expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -285,6 +292,10 @@ module API end end + class DiffRefs < Grape::Entity + expose :base_sha, :head_sha, :start_sha + end + class Commit < Grape::Entity expose :id, :short_id, :title, :created_at expose :parent_ids @@ -297,6 +308,10 @@ module API expose :additions, :deletions, :total end + class CommitWithStats < Commit + expose :stats, using: Entities::CommitStats + end + class CommitDetail < Commit expose :stats, using: Entities::CommitStats, if: :stats expose :status @@ -334,6 +349,10 @@ module API expose :developers_can_merge do |repo_branch, options| options[:project].protected_branches.developers_can?(:merge, repo_branch.name) end + + expose :can_push do |repo_branch, options| + Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name) + end end class TreeObject < Grape::Entity @@ -347,7 +366,7 @@ module API end class Snippet < Grape::Entity - expose :id, :title, :file_name, :description + expose :id, :title, :file_name, :description, :visibility expose :author, using: Entities::UserBasic expose :updated_at, :created_at expose :project_id @@ -401,10 +420,15 @@ module API expose :state, :created_at, :updated_at expose :due_date expose :start_date + + expose :web_url do |milestone, _options| + Gitlab::UrlBuilder.build(milestone) + end end class IssueBasic < ProjectEntity expose :closed_at + expose :closed_by, using: Entities::UserBasic expose :labels do |issue, options| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort @@ -547,7 +571,9 @@ module API expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch - expose :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } + expose :allow_collaboration, if: -> (merge_request, _) { merge_request.for_fork? } + # Deprecated + expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) @@ -556,6 +582,8 @@ module API expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |merge_request| merge_request end + + expose :squash end class MergeRequest < MergeRequestBasic @@ -599,6 +627,8 @@ module API merge_request.metrics&.pipeline end + expose :diff_refs, using: Entities::DiffRefs + def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end @@ -640,6 +670,11 @@ module API expose :id, :key, :created_at end + class DiffPosition < Grape::Entity + expose :base_sha, :start_sha, :head_sha, :old_path, :new_path, + :position_type + end + class Note < Grape::Entity # Only Issue and MergeRequest have iid NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze @@ -653,6 +688,14 @@ module API expose :system?, as: :system expose :noteable_id, :noteable_type + expose :position, if: ->(note, options) { note.diff_note? } do |note| + note.position.to_h + end + + expose :resolvable?, as: :resolvable + expose :resolved?, as: :resolved, if: ->(note, options) { note.resolvable? } + expose :resolved_by, using: Entities::UserBasic, if: ->(note, options) { note.resolvable? } + # Avoid N+1 queries as much as possible expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) } end @@ -663,6 +706,12 @@ module API expose :notes, using: Entities::Note end + class Avatar < Grape::Entity + expose :avatar_url do |avatarable, options| + avatarable.avatar_url(only_path: false, size: options[:size]) + end + end + class AwardEmoji < Grape::Entity expose :id expose :name @@ -792,7 +841,7 @@ module API expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :confidential_issues_events expose :merge_requests_events, :tag_push_events, :note_events - expose :pipeline_events, :wiki_page_events + expose :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events # Expose serialized properties expose :properties do |service, options| @@ -803,8 +852,8 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - if options.key?(:project_members) - (options[:project_members] || []).find { |member| member.source_id == project.id } + if options[:project_members] + options[:project_members].find { |member| member.source_id == project.id } else project.project_member(options[:current_user]) end @@ -812,8 +861,8 @@ module API expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - if options.key?(:group_members) - (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } + if options[:group_members] + options[:group_members].find { |member| member.source_id == project.namespace_id } else project.group.group_member(options[:current_user]) end @@ -824,13 +873,24 @@ module API def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - unless options.key?(:group_members) - relation = relation.preload(group: [group_members: [:source, user: [notification_settings: :source]]]) + # MySQL doesn't support LIMIT inside an IN subquery + if Gitlab::Database.mysql? + project_ids = relation.pluck('projects.id') + namespace_ids = relation.pluck(:namespace_id) + else + project_ids = relation.select('projects.id') + namespace_ids = relation.select(:namespace_id) end - unless options.key?(:project_members) - relation = relation.preload(project_members: [:source, user: [notification_settings: :source]]) - end + options[:project_members] = options[:current_user] + .project_members + .where(source_id: project_ids) + .preload(:source, user: [notification_settings: :source]) + + options[:group_members] = options[:current_user] + .group_members + .where(source_id: namespace_ids) + .preload(:source, user: [notification_settings: :source]) relation end @@ -906,8 +966,16 @@ module API end class ApplicationSetting < Grape::Entity - expose :id - expose(*::ApplicationSettingsHelper.visible_attributes) + def self.exposed_attributes + attributes = ::ApplicationSettingsHelper.visible_attributes + attributes.delete(:performance_bar_allowed_group_path) + attributes.delete(:performance_bar_enabled) + + attributes + end + + expose :id, :performance_bar_allowed_group_id + expose(*exposed_attributes) expose(:restricted_visibility_levels) do |setting, _options| setting.restricted_visibility_levels.map { |level| Gitlab::VisibilityLevel.string_level(level) } end @@ -926,7 +994,7 @@ module API end class Tag < Grape::Entity - expose :name, :message + expose :name, :message, :target expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) @@ -940,6 +1008,7 @@ module API class Runner < Grape::Entity expose :id expose :description + expose :ip_address expose :active expose :is_shared expose :name @@ -951,6 +1020,7 @@ module API expose :tag_list expose :run_untagged expose :locked + expose :maximum_timeout expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at @@ -962,6 +1032,13 @@ module API options[:current_user].authorized_projects.where(id: runner.projects) end end + expose :groups, with: Entities::BasicGroupDetails do |runner, options| + if options[:current_user].admin? + runner.groups + else + options[:current_user].authorized_groups.where(id: runner.groups) + end + end end class RunnerRegistrationDetails < Grape::Entity @@ -984,6 +1061,7 @@ module API class Job < JobBasic expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :runner, with: Runner + expose :artifacts_expire_at end class JobBasicWithProject < JobBasic @@ -1119,7 +1197,7 @@ module API end class RunnerInfo < Grape::Entity - expose :timeout + expose :metadata_timeout, as: :timeout end class Step < Grape::Entity diff --git a/lib/api/events.rb b/lib/api/events.rb index b0713ff1d54..fc4ba5a3188 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -17,6 +17,7 @@ module API def present_events(events) events = events.reorder(created_at: params[:sort]) + .with_associations present paginate(events), with: Entities::Event end diff --git a/lib/api/features.rb b/lib/api/features.rb index 9385c6ca174..11d848584d9 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -65,6 +65,13 @@ module API present feature, with: Entities::Feature, current_user: current_user end + + desc 'Remove the gate value for the given feature' + delete ':name' do + Feature.get(params[:name]).remove + + status 204 + end end end end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 92800ce6450..55d5c7f1606 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -31,7 +31,7 @@ module API key = params[:key] variable = user_group.variables.find_by(key: key) - return not_found!('GroupVariable') unless variable + break not_found!('GroupVariable') unless variable present variable, with: Entities::Variable end @@ -67,7 +67,7 @@ module API put ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) - return not_found!('GroupVariable') unless variable + break not_found!('GroupVariable') unless variable variable_params = declared_params(include_missing: false).except(:key) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 4a4df1b8b9e..c7f41aba854 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -32,23 +32,23 @@ module API optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end - def find_groups(params) - find_params = { - all_available: params[:all_available], - custom_attributes: params[:custom_attributes], - owned: params[:owned] - } - find_params[:parent] = find_group!(params[:id]) if params[:id] + def find_groups(params, parent_id = nil) + find_params = params.slice(:all_available, :custom_attributes, :owned) + find_params[:parent] = find_group!(parent_id) if parent_id + find_params[:all_available] = + find_params.fetch(:all_available, current_user&.full_private_access?) groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - groups = groups.reorder(params[:order_by] => params[:sort]) + order_options = { params[:order_by] => params[:sort] } + order_options["id"] ||= "asc" + groups = groups.reorder(order_options) groups end @@ -85,7 +85,7 @@ module API use :with_custom_attributes end get do - groups = find_groups(params) + groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups params, groups end @@ -167,9 +167,12 @@ module API group = find_group!(params[:id]) authorize! :admin_group, group + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/46285') destroy_conditionally!(group) do |group| - ::Groups::DestroyService.new(group, current_user).execute + ::Groups::DestroyService.new(group, current_user).async_execute end + + accepted! end desc 'Get a list of projects in this group.' do @@ -213,7 +216,7 @@ module API use :with_custom_attributes end get ":id/subgroups" do - groups = find_groups(params) + groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups params, groups end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e4fca77ab5d..9c53b7c3fe7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -83,12 +83,13 @@ module API end def available_labels_for(label_parent) - search_params = - if label_parent.is_a?(Project) - { project_id: label_parent.id } - else - { group_id: label_parent.id, only_group_labels: true } - end + search_params = { include_ancestor_groups: true } + + if label_parent.is_a?(Project) + search_params[:project_id] = label_parent.id + else + search_params.merge!(group_id: label_parent.id, only_group_labels: true) + end LabelsFinder.new(current_user, search_params).execute end @@ -102,9 +103,9 @@ module API end def find_project(id) - if id =~ /^\d+$/ + if id.is_a?(Integer) || id =~ /^\d+$/ Project.find_by(id: id) - else + elsif id.include?("/") Project.find_by_full_path(id) end end @@ -170,6 +171,10 @@ module API MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end + def find_project_commit(id) + user_project.commit_by(oid: id) + end + def find_project_snippet(id) finder_params = { project: user_project } SnippetsFinder.new(current_user, finder_params).find(id) @@ -267,7 +272,8 @@ module API attrs[key] = params_hash[key] end end - ActionController::Parameters.new(attrs).permit! + permitted_attrs = ActionController::Parameters.new(attrs).permit! + Gitlab.rails5? ? permitted_attrs.to_h : permitted_attrs end def filter_by_iid(items, iid) @@ -388,29 +394,7 @@ module API # file helpers - def uploaded_file(field, uploads_path) - if params[field] - bad_request!("#{field} is not a file") unless params[field][:filename] - return params[field] - end - - return nil unless params["#{field}.path"] && params["#{field}.name"] - - # sanitize file paths - # this requires all paths to exist - required_attributes! %W(#{field}.path) - uploads_path = File.realpath(uploads_path) - file_path = File.realpath(params["#{field}.path"]) - bad_request!('Bad file path') unless file_path.start_with?(uploads_path) - - UploadedFile.new( - file_path, - params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream' - ) - end - - def present_file!(path, filename, content_type = 'application/octet-stream') + def present_disk_file!(path, filename, content_type = 'application/octet-stream') filename ||= File.basename(path) header['Content-Disposition'] = "attachment; filename=#{filename}" header['Content-Transfer-Encoding'] = 'binary' @@ -426,13 +410,17 @@ module API end end - def present_artifacts!(artifacts_file) - return not_found! unless artifacts_file.exists? + def present_carrierwave_file!(file, supports_direct_download: true) + return not_found! unless file.exists? - if artifacts_file.file_storage? - present_file!(artifacts_file.path, artifacts_file.filename) + if file.file_storage? + present_disk_file!(file.path, file.filename) + elsif supports_direct_download && file.class.direct_download_enabled? + redirect(file.url) else - redirect_to(artifacts_file.url) + header(*Gitlab::Workhorse.send_url(file.url)) + status :ok + body end end @@ -485,8 +473,8 @@ module API header(*Gitlab::Workhorse.send_git_blob(repository, blob)) end - def send_git_archive(repository, ref:, format:) - header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) + def send_git_archive(repository, **kwargs) + header(*Gitlab::Workhorse.send_git_archive(repository, **kwargs)) end def send_artifacts_entry(build, entry) diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb index 70e4eda95f8..10d652e33f5 100644 --- a/lib/api/helpers/custom_attributes.rb +++ b/lib/api/helpers/custom_attributes.rb @@ -7,6 +7,9 @@ module API helpers do params :with_custom_attributes do optional :with_custom_attributes, type: Boolean, default: false, desc: 'Include custom attributes in the response' + + optional :custom_attributes, type: Hash, + desc: 'Filter with custom attributes' end def with_custom_attributes(collection_or_resource, options = {}) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 14648588dfd..83151be82ad 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -29,18 +29,6 @@ module API {} end - def fix_git_env_repository_paths(env, repository_path) - if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence - env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) - end - - if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) } - end - - env - end - def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS @@ -101,12 +89,6 @@ module API end end - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - def repository_path - repository.path_to_repo - end - # Return the Gitaly Address if it is enabled def gitaly_payload(action) return unless %w[git-receive-pack git-upload-pack git-upload-archive].include?(action) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index cd91df1ecd8..b4bfb677d72 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -21,6 +21,23 @@ module API end end + def resolve_note(noteable, note_id, resolved) + note = noteable.notes.find(note_id) + + authorize! :resolve_note, note + + bad_request!("Note is not resolvable") unless note.resolvable? + + if resolved + parent = noteable_parent(noteable) + ::Notes::ResolveService.new(parent, current_user).execute(note) + else + note.unresolve! + end + + present note, with: Entities::Note + end + def delete_note(noteable, note_id) note = noteable.notes.find(note_id) @@ -35,7 +52,7 @@ module API def get_note(noteable, note_id) note = noteable.notes.with_metadata.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) + can_read_note = !note.cross_reference_not_visible_for?(current_user) if can_read_note present note, with: Entities::Note @@ -49,7 +66,20 @@ module API end def find_noteable(parent, noteables_str, noteable_id) - public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend + noteable = public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend + + readable = + if noteable.is_a?(Commit) + # for commits there is not :read_commit policy, check if user + # has :read_note permission on the commit's project + can?(current_user, :read_note, user_project) + else + can?(current_user, noteable_read_ability_name(noteable), noteable) + end + + return not_found!(noteables_str) unless readable + + noteable end def noteable_parent(noteable) @@ -57,20 +87,34 @@ module API end def create_note(noteable, opts) - noteables_str = noteable.model_name.to_s.underscore.pluralize - - return not_found!(noteables_str) unless can?(current_user, noteable_read_ability_name(noteable), noteable) - - authorize! :create_note, noteable + policy_object = noteable.is_a?(Commit) ? user_project : noteable + authorize!(:create_note, policy_object) parent = noteable_parent(noteable) + if opts[:created_at] - opts.delete(:created_at) unless current_user.admin? || parent.owner == current_user + opts.delete(:created_at) unless + current_user.admin? || parent.owned_by?(current_user) end project = parent if parent.is_a?(Project) ::Notes::CreateService.new(project, current_user, opts).execute end + + def resolve_discussion(noteable, discussion_id, resolved) + discussion = noteable.find_discussion(discussion_id) + + forbidden! unless discussion.can_resolve?(current_user) + + if resolved + parent = noteable_parent(noteable) + ::Discussions::ResolveService.new(parent, current_user, merge_request: noteable).execute(discussion) + else + discussion.unresolve! + end + + present discussion, with: Entities::Discussion + end end end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 09805049169..3308212216e 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -2,67 +2,240 @@ module API module Helpers module Pagination def paginate(relation) - relation = add_default_order(relation) + strategy = if params[:pagination] == 'keyset' && Feature.enabled?('api_keyset_pagination') + KeysetPaginationStrategy + else + DefaultPaginationStrategy + end - relation.page(params[:page]).per(params[:per_page]).tap do |data| - add_pagination_headers(data) - end + strategy.new(self).paginate(relation) end - private + class KeysetPaginationInfo + attr_reader :relation, :request_context - def add_pagination_headers(paginated_data) - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) + def initialize(relation, request_context) + # This is because it's rather complex to support multiple values with possibly different sort directions + # (and we don't need this in the API) + if relation.order_values.size > 1 + raise "Pagination only supports ordering by a single column." \ + "The following columns were given: #{relation.order_values.map { |v| v.expr.name }}" + end - return if data_without_counts?(paginated_data) + @relation = relation + @request_context = request_context + end - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s - end + def fields + keys.zip(values).reject { |_, v| v.nil? }.to_h + end - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value + def column_for_order_by(relation) + relation.order_values.first&.expr&.name + end - links = [] + # Sort direction (`:asc` or `:desc`) + def sort + @sort ||= if order_by_primary_key? + # Default order is by id DESC + :desc + else + # API defaults to DESC order if param `sort` not present + request_context.params[:sort]&.to_sym || :desc + end + end - request_params[:page] = paginated_data.prev_page - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page] + # Do we only sort by primary key? + def order_by_primary_key? + keys.size == 1 && keys.first == primary_key + end - request_params[:page] = paginated_data.next_page - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page] + def primary_key + relation.model.primary_key.to_sym + end - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + def sort_ascending? + sort == :asc + end - unless data_without_counts?(paginated_data) - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + # Build hash of request parameters for a given record (relevant to pagination) + def params_for(record) + return {} unless record + + keys.each_with_object({}) do |key, h| + h["ks_prev_#{key}".to_sym] = record.attributes[key.to_s] + end end - links.join(', ') - end + private + + # All values present in request parameters that correspond to #keys. + def values + @values ||= keys.map do |key| + request_context.params["ks_prev_#{key}".to_sym] + end + end - def total_pages(paginated_data) - # Ensure there is in total at least 1 page - [paginated_data.total_pages, 1].max + # All keys relevant to pagination. + # This always includes the primary key. Optionally, the `order_by` key is prepended. + def keys + @keys ||= [column_for_order_by(relation), primary_key].compact.uniq + end end - def add_default_order(relation) - if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? - relation = relation.order(:id) + class KeysetPaginationStrategy + attr_reader :request_context + delegate :params, :header, :request, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def paginate(relation) + pagination = KeysetPaginationInfo.new(relation, request_context) + + paged_relation = relation.limit(per_page) + + if conds = conditions(pagination) + paged_relation = paged_relation.where(*conds) + end + + # In all cases: sort by primary key (possibly in addition to another sort column) + paged_relation = paged_relation.order(pagination.primary_key => pagination.sort) + + add_default_pagination_headers + + if last_record = paged_relation.last + next_page_params = pagination.params_for(last_record) + add_navigation_links(next_page_params) + end + + paged_relation + end + + private + + def conditions(pagination) + fields = pagination.fields + + return nil if fields.empty? + + placeholder = fields.map { '?' } + + comp = if pagination.sort_ascending? + '>' + else + '<' + end + + [ + # Row value comparison: + # (A, B) < (a, b) <=> (A < a) OR (A = a AND B < b) + # <=> A <= a AND ((A < a) OR (A = a AND B < b)) + "(#{fields.keys.join(',')}) #{comp} (#{placeholder.join(',')})", + *fields.values + ] + end + + def per_page + params[:per_page] + end + + def add_default_pagination_headers + header 'X-Per-Page', per_page.to_s + end + + def add_navigation_links(next_page_params) + header 'X-Next-Page', page_href(next_page_params) + header 'Link', link_for('next', next_page_params) end - relation + def page_href(next_page_params) + request_url = request.url.split('?').first + request_params = params.dup + request_params[:per_page] = per_page + + request_params.merge!(next_page_params) if next_page_params + + "#{request_url}?#{request_params.to_query}" + end + + def link_for(rel, next_page_params) + %(<#{page_href(next_page_params)}>; rel="#{rel}") + end end - def data_without_counts?(paginated_data) - paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + class DefaultPaginationStrategy + attr_reader :request_context + delegate :params, :header, :request, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def paginate(relation) + relation = add_default_order(relation) + + relation.page(params[:page]).per(params[:per_page]).tap do |data| + add_pagination_headers(data) + end + end + + private + + def add_default_order(relation) + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? + relation = relation.order(:id) + end + + relation + end + + def add_pagination_headers(paginated_data) + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.prev_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page] + + request_params[:page] = paginated_data.next_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page] + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + unless data_without_counts?(paginated_data) + request_params[:page] = total_pages(paginated_data) + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + end + + links.join(', ') + end + + def total_pages(paginated_data) + # Ensure there is in total at least 1 page + [paginated_data.total_pages, 1].max + end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end end end end diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb new file mode 100644 index 00000000000..94798a8cb51 --- /dev/null +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -0,0 +1,25 @@ +module API + module Helpers + module ProjectSnapshotsHelpers + def authorize_read_git_snapshot! + authenticated_with_full_private_access! + end + + def send_git_snapshot(repository) + header(*Gitlab::Workhorse.send_git_snapshot(repository)) + end + + def snapshot_project + user_project + end + + def snapshot_repository + if to_boolean(params[:wiki]) + snapshot_project.wiki.repository + else + snapshot_project.repository + end + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb new file mode 100644 index 00000000000..381d5e8968c --- /dev/null +++ b/lib/api/helpers/projects_helpers.rb @@ -0,0 +1,38 @@ +module API + module Helpers + module ProjectsHelpers + extend ActiveSupport::Concern + + included do + helpers do + params :optional_project_params_ce do + optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' + optional :avatar, type: File, desc: 'Avatar image for project' + optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' + optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + end + + params :optional_project_params do + use :optional_project_params_ce + end + end + end + end + end +end diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index 7f4d6e58b34..bc7333ca4b3 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -1,7 +1,7 @@ module API module Helpers module RelatedResourcesHelpers - include GrapeRouteHelpers::NamedRouteMatcher + include GrapePathHelpers::NamedRouteMatcher def issues_available?(project, options) available?(:issues, project, options[:current_user]) @@ -13,9 +13,14 @@ module API def expose_url(path) url_options = Gitlab::Application.routes.default_url_options - protocol, host, port = url_options.slice(:protocol, :host, :port).values + protocol, host, port, script_name = url_options.values_at(:protocol, :host, :port, :script_name) - URI::Generic.build(scheme: protocol, host: host, port: port, path: path).to_s + # Using a blank component at the beginning of the join we ensure + # that the resulted path will start with '/'. If the resulted path + # does not start with '/', URI::Generic#build will fail + path_with_script_name = File.join('', [script_name, path].select(&:present?)) + + URI::Generic.build(scheme: protocol, host: host, port: port, path: path_with_script_name).to_s end private diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 35ac0b4cbca..61eb88d3331 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -59,6 +59,11 @@ module API def max_artifacts_size Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i end + + def job_forbidden!(job, reason) + header 'Job-Status', job.status + forbidden!(reason) + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index b3660e4a1d0..a9803be9f69 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -21,8 +21,7 @@ module API # Stores some Git-specific env thread-safely env = parse_env - env = fix_git_env_repository_paths(env, repository_path) if project - Gitlab::Git::Env.set(env) + Gitlab::Git::HookEnv.set(gl_repository, env) if project actor = if params[:key_id] @@ -51,7 +50,7 @@ module API access_checker.check(params[:action], params[:changes]) @project ||= access_checker.project rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e - return { status: false, message: e.message } + break { status: false, message: e.message } end log_user_activity(actor) @@ -60,7 +59,11 @@ module API status: true, gl_repository: gl_repository, gl_username: user&.username, - repository_path: repository_path, + + # This repository_path is a bogus value but gitlab-shell still requires + # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 + repository_path: '/', + gitaly: gitaly_payload(params[:action]) } end @@ -114,7 +117,7 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION, + gitlab_rev: Gitlab.revision, redis: redis_ping } end @@ -143,21 +146,21 @@ module API if key key.update_last_used_at else - return { 'success' => false, 'message' => 'Could not find the given key' } + break { 'success' => false, 'message' => 'Could not find the given key' } end if key.is_a?(DeployKey) - return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } end user = key.user unless user - return { success: false, message: 'Could not find a user for the given key' } + break { success: false, message: 'Could not find a user for the given key' } end unless user.two_factor_enabled? - return { success: false, message: 'Two-factor authentication is not enabled for this user' } + break { success: false, message: 'Two-factor authentication is not enabled for this user' } end codes = nil diff --git a/lib/api/issues.rb b/lib/api/issues.rb index f74b3b26802..25185d6edc8 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -2,7 +2,7 @@ module API class Issues < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } helpers ::Gitlab::IssuableMetadata @@ -13,9 +13,10 @@ module API args.delete(:id) args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) + args[:scope] = args[:scope].underscore if args[:scope] issues = IssuesFinder.new(current_user, args).execute - .preload(:assignees, :labels, :notes, :timelogs) + .preload(:assignees, :labels, :notes, :timelogs, :project, :author) issues.reorder(args[:order_by] => args[:sort]) end @@ -36,8 +37,8 @@ module API optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' - optional :scope, type: String, values: %w[created-by-me assigned-to-me all], - desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end @@ -66,10 +67,11 @@ module API optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' use :issues_params - optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', - desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me', + desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' end get do + authenticate! unless params[:scope] == 'all' issues = paginate(find_issues) options = { @@ -97,7 +99,7 @@ module API get ":id/issues" do group = find_group!(params[:id]) - issues = paginate(find_issues(group_id: group.id)) + issues = paginate(find_issues(group_id: group.id, include_subgroups: true)) options = { with: Entities::IssueBasic, @@ -310,7 +312,7 @@ module API issue = find_project_issue(params[:issue_iid]) - return not_found!('UserAgentDetail') unless issue.user_agent_detail + break not_found!('UserAgentDetail') unless issue.user_agent_detail present issue.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 47e5eeab31d..32379d7c8ab 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -28,7 +28,7 @@ module API builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) - present_artifacts!(latest_build.artifacts_file) + present_carrierwave_file!(latest_build.artifacts_file) end desc 'Download the artifacts archive from a job' do @@ -43,7 +43,7 @@ module API build = find_build!(params[:job_id]) - present_artifacts!(build.artifacts_file) + present_carrierwave_file!(build.artifacts_file) end desc 'Download a specific file from artifacts archive' do @@ -77,7 +77,7 @@ module API build = find_build!(params[:job_id]) authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? + break not_found!(build) unless build.artifacts? build.keep_artifacts! diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 9c205514b3a..e95b0dd5267 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -54,6 +54,7 @@ module API pipeline = user_project.pipelines.find(params[:pipeline_id]) builds = pipeline.builds builds = filter_builds(builds, params[:scope]) + builds = builds.preload(:job_artifacts_archive) present paginate(builds), with: Entities::Job end @@ -72,7 +73,7 @@ module API present build, with: Entities::Job end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. desc 'Get a trace of a specific job of a project' @@ -120,7 +121,7 @@ module API build = find_build!(params[:job_id]) authorize!(:update_build, build) - return forbidden!('Job is not retryable') unless build.retryable? + break forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -138,7 +139,7 @@ module API build = find_build!(params[:job_id]) authorize!(:erase_build, build) - return forbidden!('Job is not erasable!') unless build.erasable? + break forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) present build, with: Entities::Job diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb new file mode 100644 index 00000000000..5d55224c1a7 --- /dev/null +++ b/lib/api/markdown.rb @@ -0,0 +1,31 @@ +module API + class Markdown < Grape::API + params do + requires :text, type: String, desc: "The markdown text to render" + optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown" + optional :project, type: String, desc: "The full path of a project to use as the context when creating references using GitLab Flavored Markdown" + end + resource :markdown do + desc "Render markdown text" do + detail "This feature was introduced in GitLab 11.0." + end + post do + context = { only_path: false } + + if params[:project] + project = Project.find_by_full_path(params[:project]) + + not_found!("Project") unless can?(current_user, :read_project, project) + + context[:project] = project + else + context[:skip_project_check] = true + end + + context[:pipeline] = params[:gfm] ? :full : :plain_markdown + + { html: Banzai.render(params[:text], context) } + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3264a26f7d2..af7d2471b34 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -10,12 +10,6 @@ module API helpers do params :optional_params_ee do end - - params :merge_params_ee do - end - - def update_merge_request_ee(merge_request) - end end def self.update_params_at_least_one_of @@ -29,6 +23,7 @@ module API target_branch title discussion_locked + squash ] end @@ -38,11 +33,12 @@ module API args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) + args[:scope] = args[:scope].underscore if args[:scope] merge_requests = MergeRequestsFinder.new(current_user, args).execute .reorder(args[:order_by] => args[:sort]) merge_requests = paginate(merge_requests) - .preload(:target_project) + .preload(:source_project, :target_project) return merge_requests if args[:view] == 'simple' @@ -63,6 +59,18 @@ module API end end + def serializer_options_for(merge_requests) + options = { with: Entities::MergeRequestBasic, current_user: current_user } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + options + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', desc: 'Return opened, closed, merged, or all merge requests' @@ -79,8 +87,8 @@ module API optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' - optional :scope, type: String, values: %w[created-by-me assigned-to-me all], - desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' @@ -95,23 +103,33 @@ module API end params do use :merge_requests_params - optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me', - desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me', + desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' end get do authenticate! unless params[:scope] == 'all' merge_requests = find_merge_requests - options = { with: Entities::MergeRequestBasic, - current_user: current_user } + present merge_requests, serializer_options_for(merge_requests) + end + end - if params[:view] == 'simple' - options[:with] = Entities::MergeRequestSimple - else - options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') - end + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Get a list of group merge requests' do + success Entities::MergeRequestBasic + end + params do + use :merge_requests_params + end + get ":id/merge_requests" do + group = find_group!(params[:id]) - present merge_requests, options + merge_requests = find_merge_requests(group_id: group.id, include_subgroups: true) + + present merge_requests, serializer_options_for(merge_requests) end end @@ -144,7 +162,9 @@ module API optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - optional :allow_maintainer_to_push, type: Boolean, desc: 'Whether a maintainer of the target project can push to the source project' + optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch' + optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration' + optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' use :optional_params_ee end @@ -162,15 +182,8 @@ module API merge_requests = find_merge_requests(project_id: user_project.id) - options = { with: Entities::MergeRequestBasic, - current_user: current_user, - project: user_project } - - if params[:view] == 'simple' - options[:with] = Entities::MergeRequestSimple - else - options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') - end + options = serializer_options_for(merge_requests) + options[:project] = user_project present merge_requests, options end @@ -189,7 +202,7 @@ module API post ":id/merge_requests" do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316') - authorize! :create_merge_request, user_project + authorize! :create_merge_request_from, user_project mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) @@ -307,8 +320,7 @@ module API optional :merge_when_pipeline_succeeds, type: Boolean, desc: 'When true, this merge request will be merged when the pipeline succeeds' optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' - - use :merge_params_ee + optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' end put ':id/merge_requests/:merge_request_iid/merge' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') @@ -326,7 +338,7 @@ module API check_sha_param!(params, merge_request) - update_merge_request_ee(merge_request) + merge_request.update(squash: params[:squash]) if params[:squash] merge_params = { commit_message: params[:merge_commit_message], diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index c570eace862..a8eb137e46a 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -24,7 +24,7 @@ module API optional :state_event, type: String, values: %w[close activate], desc: 'The state event of the milestone ' use :optional_params - at_least_one_of :title, :description, :due_date, :state_event + at_least_one_of :title, :description, :start_date, :due_date, :state_event end def list_milestones_for(parent) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 69f1df6b341..39923e6d5b5 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -31,23 +31,19 @@ module API get ":id/#{noteables_str}/:noteable_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - if can?(current_user, noteable_read_ability_name(noteable), noteable) - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(raw_notes) - .reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note - else - not_found!("Notes") - end + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(raw_notes) + .reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note end desc "Get a single #{noteable_type.to_s.downcase} note" do diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index d2b8b832e4e..8374a57edfa 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -19,6 +19,7 @@ module API optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, desc: 'The status of pipelines' optional :ref, type: String, desc: 'The ref of pipelines' + optional :sha, type: String, desc: 'The sha of pipelines' optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' optional :name, type: String, desc: 'The name of the user who triggered pipelines' optional :username, type: String, desc: 'The username of the user who triggered pipelines' @@ -40,15 +41,20 @@ module API end params do requires :ref, type: String, desc: 'Reference' + optional :variables, Array, desc: 'Array of variables available in the pipeline' end post ':id/pipeline' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') authorize! :create_pipeline, user_project + pipeline_params = declared_params(include_missing: false) + .merge(variables_attributes: params[:variables]) + .except(:variables) + new_pipeline = Ci::CreatePipelineService.new(user_project, current_user, - declared_params(include_missing: false)) + pipeline_params) .execute(:api, ignore_skip_ci: true, save_on_errors: false) if new_pipeline.persisted? diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index b0a7fd6f4ab..5ef4e9d530c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API render_api_error!('404 Not found or has expired', 404) unless path - present_file!(path, File.basename(path), 'application/gzip') + present_disk_file!(path, File.basename(path), 'application/gzip') end desc 'Start export' do @@ -33,11 +33,28 @@ module API end params do optional :description, type: String, desc: 'Override the project description' + optional :upload, type: Hash do + optional :url, type: String, desc: 'The URL to upload the project' + optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project' + end end post ':id/export' do project_export_params = declared_params(include_missing: false) + after_export_params = project_export_params.delete(:upload) || {} - user_project.add_export_job(current_user: current_user, params: project_export_params) + export_strategy = if after_export_params[:url].present? + params = after_export_params.slice(:url, :http_method).symbolize_keys + + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + end + + if export_strategy&.invalid? + render_validation_error!(export_strategy) + else + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + end accepted! end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index f82241058e5..68921ae439b 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -14,6 +14,7 @@ module API optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :confidential_note_events, type: Boolean, desc: "Trigger hook on confidential note(comment) events" optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index a509c1f32c1..bc5152e539f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,6 +1,7 @@ module API class ProjectImport < Grape::API include PaginationParams + include Helpers::ProjectsHelpers helpers do def import_params @@ -25,6 +26,12 @@ module API requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' + optional :override_params, + type: Hash, + desc: 'New project params to override values in the export' do + use :optional_project_params + end end desc 'Create a new project import' do detail 'This feature was introduced in GitLab 10.6.' @@ -44,10 +51,15 @@ module API project_params = { path: import_params[:path], namespace_id: namespace.id, - file: import_params[:file]['tempfile'] + file: import_params[:file]['tempfile'], + overwrite: import_params[:overwrite] } - project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute + override_params = import_params.delete(:override_params) + + project = ::Projects::GitlabProjectsImportService.new( + current_user, project_params, override_params + ).execute render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb new file mode 100644 index 00000000000..71005acc587 --- /dev/null +++ b/lib/api/project_snapshots.rb @@ -0,0 +1,19 @@ +module API + class ProjectSnapshots < Grape::API + helpers ::API::Helpers::ProjectSnapshotsHelpers + + before { authorize_read_git_snapshot! } + + resource :projects do + desc 'Download a (possibly inconsistent) snapshot of a repository' do + detail 'This feature was introduced in GitLab 10.7' + end + params do + optional :wiki, type: Boolean, desc: 'Set to true to receive the wiki repository' + end + get ':id/snapshot' do + send_git_snapshot(snapshot_repository) + end + end + end +end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 39c03c40bab..1de5551fee9 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -145,7 +145,7 @@ module API snippet = Snippet.find_by!(id: params[:snippet_id], project_id: params[:id]) - return not_found!('UserAgentDetail') unless snippet.user_agent_detail + break not_found!('UserAgentDetail') unless snippet.user_agent_detail present snippet.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index b552b0e0c5d..3ef3680c5d9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -4,36 +4,11 @@ module API class Projects < Grape::API include PaginationParams include Helpers::CustomAttributes + include Helpers::ProjectsHelpers before { authenticate_non_get! } helpers do - params :optional_params_ce do - optional :description, type: String, desc: 'The description of the project' - optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' - optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' - optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' - optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' - optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' - optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' - optional :public_builds, type: Boolean, desc: 'Perform public builds' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - optional :tag_list, type: Array[String], desc: 'The list of tags for a project' - optional :avatar, type: File, desc: 'Avatar image for project' - optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' - end - - params :optional_params do - use :optional_params_ce - end - params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end @@ -83,22 +58,20 @@ module API projects = paginate(projects) projects, options = with_custom_attributes(projects, options) - if current_user - project_members = current_user.project_members.preload(:source, user: [notification_settings: :source]) - group_members = current_user.group_members.preload(:source, user: [notification_settings: :source]) - end - options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - project_members: project_members, - group_members: group_members, current_user: current_user ) options[:with] = Entities::BasicProjectDetails if params[:simple] present options[:with].prepare_relation(projects, options), options end + + def translate_params_for_compatibility(params) + params[:builds_enabled] = params.delete(:jobs_enabled) if params.key?(:jobs_enabled) + params + end end resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do @@ -143,12 +116,12 @@ module API optional :name, type: String, desc: 'The name of the project' optional :path, type: String, desc: 'The path of the repository' at_least_one_of :name, :path - use :optional_params + use :optional_project_params use :create_params end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) + attrs = translate_params_for_compatibility(attrs) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -171,7 +144,7 @@ module API requires :user_id, type: Integer, desc: 'The ID of a user' optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' - use :optional_params + use :optional_project_params use :create_params end post "user/:user_id" do @@ -180,6 +153,7 @@ module API not_found!('User') unless user attrs = declared_params(include_missing: false) + attrs = translate_params_for_compatibility(attrs) project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -228,11 +202,7 @@ module API namespace_id = fork_params[:namespace] if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end + fork_params[:namespace] = find_namespace(namespace_id) unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') @@ -278,6 +248,7 @@ module API :issues_enabled, :lfs_enabled, :merge_requests_enabled, + :merge_method, :name, :only_allow_merge_if_all_discussions_are_resolved, :only_allow_merge_if_pipeline_succeeds, @@ -295,7 +266,7 @@ module API optional :default_branch, type: String, desc: 'The default branch of the project' optional :path, type: String, desc: 'The path of the repository' - use :optional_params + use :optional_project_params at_least_one_of(*at_least_one_of_ce) end put ':id' do @@ -304,7 +275,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) + attrs = translate_params_for_compatibility(attrs) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute @@ -366,6 +337,11 @@ module API end end + desc 'Get languages in project repository' + get ':id/languages' do + user_project.repository.languages.map { |language| language.values_at(:label, :value) }.to_h + end + desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project @@ -425,7 +401,7 @@ module API end unless user_project.allowed_to_share_with_group? - return render_api_error!("The project sharing with group is disabled", 400) + break render_api_error!("The project sharing with group is disabled", 400) end link = user_project.project_group_links.new(declared_params(include_missing: false)) diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index c15c487deb4..a30eb46c220 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -41,10 +41,10 @@ module API requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, - desc: 'Access levels allowed to push (defaults: `40`, master access level)' + desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)' optional :merge_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, - desc: 'Access levels allowed to merge (defaults: `40`, master access level)' + desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' end post ':id/protected_branches' do protected_branch = user_project.protected_branches.find_by(name: params[:name]) @@ -52,11 +52,7 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - # Replace with `declared(params)` after updating to grape v1.0.2 - # See https://github.com/ruby-grape/grape/pull/1710 - # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 - declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - + declared_params = declared_params(include_missing: false) api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) protected_branch = api_service.create @@ -74,7 +70,10 @@ module API delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) - destroy_conditionally!(protected_branch) + destroy_conditionally!(protected_branch) do + destroy_service = ::ProtectedBranches::DestroyService.new(user_project, current_user) + destroy_service.execute(protected_branch) + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 9638c53a1df..bb3fa99af38 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -88,7 +88,7 @@ module API end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin - send_git_archive user_project.repository, ref: params[:sha], format: params[:format] + send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true rescue not_found!('File') end @@ -111,8 +111,8 @@ module API end params do use :pagination - optional :order_by, type: String, values: %w[email name commits], default: nil, desc: 'Return contributors ordered by `name` or `email` or `commits`' - optional :sort, type: String, values: %w[asc desc], default: nil, desc: 'Sort by asc (ascending) or desc (descending)' + optional :order_by, type: String, values: %w[email name commits], default: 'commits', desc: 'Return contributors ordered by `name` or `email` or `commits`' + optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' end get ':id/repository/contributors' do begin diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 7e6c33ec33d..96a02914faa 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -11,29 +11,36 @@ module API requires :token, type: String, desc: 'Registration token' optional :description, type: String, desc: %q(Runner's description) optional :info, type: Hash, desc: %q(Runner's metadata) + optional :active, type: Boolean, desc: 'Should Runner be active' optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs' optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) + optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end post '/' do - attributes = attributes_for_keys([:description, :locked, :run_untagged, :tag_list]) + attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) - runner = + attributes = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create(attributes.merge(is_shared: true)) + attributes.merge(is_shared: true, runner_type: :instance_type) elsif project = Project.find_by(runners_token: params[:token]) - # Create a specific runner for project. - project.runners.create(attributes) + # Create a specific runner for the project + attributes.merge(is_shared: false, runner_type: :project_type, projects: [project]) + elsif group = Group.find_by(runners_token: params[:token]) + # Create a specific runner for the group + attributes.merge(is_shared: false, runner_type: :group_type, groups: [group]) + else + forbidden! end - return forbidden! unless runner + runner = Ci::Runner.create(attributes) - if runner.id + if runner.persisted? present runner, with: Entities::RunnerRegistrationDetails else - not_found! + render_validation_error!(runner) end end @@ -77,12 +84,16 @@ module API end post '/request' do authenticate_runner! - no_content! unless current_runner.active? + + unless current_runner.active? + header 'X-GitLab-Last-Update', current_runner.ensure_runner_queue_value + break no_content! + end if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) - return no_content! + break no_content! end new_update = current_runner.ensure_runner_queue_value @@ -118,6 +129,7 @@ module API end put '/:id' do job = authenticate_job! + job_forbidden!(job, 'Job is not running') unless job.running? job.trace.set(params[:trace]) if params[:trace] @@ -125,10 +137,12 @@ module API project: job.project.full_path) case params[:state].to_s + when 'running' + job.touch if job.needs_touch? when 'success' - job.success + job.success! when 'failed' - job.drop(params[:failure_reason] || :unknown_failure) + job.drop!(params[:failure_reason] || :unknown_failure) end end @@ -144,14 +158,26 @@ module API end patch '/:id/trace' do job = authenticate_job! + job_forbidden!(job, 'Job is not running') unless job.running? error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') - stream_size = job.trace.append(request.body.read, content_range[0].to_i) - if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) + # TODO: + # it seems that `Content-Range` as formatted by runner is wrong, + # the `byte_end` should point to final byte, but it points byte+1 + # that means that we have to calculate end of body, + # as we cannot use `content_length[1]` + # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275 + + body_data = request.body.read + body_start = content_range[0].to_i + body_end = body_start + body_data.bytesize + + stream_size = job.trace.append(body_data, body_start) + unless stream_size == body_end + break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{stream_size}" }) end status 202 @@ -185,7 +211,7 @@ module API status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok + JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size) end desc 'Upload artifacts for job' do @@ -200,13 +226,15 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) - optional :file, type: File, desc: %q(Artifact's file) optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) - optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) + optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse)) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse)) + optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse)) end post '/:id/artifacts' do not_allowed! unless Gitlab.config.artifacts.enabled @@ -215,21 +243,34 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - workhorse_upload_path = JobArtifactUploader.workhorse_upload_path - artifacts = uploaded_file(:file, workhorse_upload_path) - metadata = uploaded_file(:metadata, workhorse_upload_path) + artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path) + metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size + bad_request!("Already uploaded") if job.job_artifacts_archive + expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) - job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata - job.artifacts_expire_in = expire_in + job.build_job_artifacts_archive( + project: job.project, + file: artifacts, + file_type: :archive, + file_sha256: artifacts.sha256, + expire_in: expire_in) + + if metadata + job.build_job_artifacts_metadata( + project: job.project, + file: metadata, + file_type: :metadata, + file_sha256: metadata.sha256, + expire_in: expire_in) + end - if job.save + if job.update(artifacts_expire_in: expire_in) present job, with: Entities::JobRequest::Response else render_validation_error!(job) @@ -244,11 +285,12 @@ module API params do requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) + optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end get '/:id/artifacts' do job = authenticate_job! - present_artifacts!(job.artifacts_file) + present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 996457c5dfe..2b78075ddbf 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -14,7 +14,7 @@ module API use :pagination end get do - runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared)) + runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared)) present paginate(runners), with: Entities::Runner end @@ -57,6 +57,7 @@ module API optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' optional :access_level, type: String, values: Ci::Runner.access_levels.keys, desc: 'The access_level of the runner' + optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do @@ -132,12 +133,10 @@ module API runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) - runner_project = runner.assign_to(user_project) - - if runner_project.persisted? + if runner.assign_to(user_project) present runner, with: Entities::Runner else - conflict!("Runner was already enabled for this project") + render_validation_error!(runner) end end @@ -183,40 +182,35 @@ module API def authenticate_show_runner!(runner) return if runner.is_shared || current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end def authenticate_update_runner!(runner) return if current_user.admin? - forbidden!("Runner is shared") if runner.is_shared? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :update_runner, runner) end def authenticate_delete_runner!(runner) return if current_user.admin? - forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :delete_runner, runner) end def authenticate_enable_runner!(runner) - forbidden!("Runner is shared") if runner.is_shared? - forbidden!("Runner is locked") if runner.locked? + forbidden!("Runner is a group runner") if runner.group_type? + return if current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("Runner is locked") if runner.locked? + forbidden!("No access granted") unless can?(current_user, :assign_runner, runner) end def authenticate_list_runners_jobs!(runner) return if current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) - end - - def user_can_access_runner?(runner) - current_user.ci_authorized_runners.exists?(runner.id) + forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end end end diff --git a/lib/api/search.rb b/lib/api/search.rb index 5d9ec617cb7..37fbabe419c 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -34,9 +34,7 @@ module API def process_results(results) case params[:scope] - when 'wiki_blobs' - paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) } - when 'blobs' + when 'blobs', 'wiki_blobs' paginate(results).map { |blob| blob[1] } else paginate(results) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 152df23a327..02ef89f997f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -5,7 +5,7 @@ module API helpers do def current_settings @current_setting ||= - (ApplicationSetting.current || ApplicationSetting.create_from_defaults) + (ApplicationSetting.current_without_cache || ApplicationSetting.create_from_defaults) end end @@ -24,7 +24,7 @@ module API optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' @@ -49,6 +49,9 @@ module API optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' + optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.' + optional :performance_bar_allowed_group_id, type: String, desc: 'Depreated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6 + optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6 optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -126,6 +129,7 @@ module API optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' + optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -134,12 +138,25 @@ module API desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." end - optional(*::ApplicationSettingsHelper.visible_attributes) - at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) + optional_attributes = ::ApplicationSettingsHelper.visible_attributes << :performance_bar_allowed_group_id + + optional(*optional_attributes) + at_least_one_of(*optional_attributes) end put "application/settings" do attrs = declared_params(include_missing: false) + # support legacy names, can be removed in v6 + if attrs.has_key?(:performance_bar_allowed_group_id) + attrs[:performance_bar_allowed_group_path] = attrs.delete(:performance_bar_allowed_group_id) + end + + # support legacy names, can be removed in v6 + if attrs.has_key?(:performance_bar_enabled) + performance_bar_enabled = attrs.delete(:performance_bar_allowed_group_id) + attrs[:performance_bar_allowed_group_path] = nil unless performance_bar_enabled + end + # support legacy names, can be removed in v5 if attrs.has_key?(:signin_enabled) attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c736cc32021..b30305b4bc9 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -94,7 +94,7 @@ module API end put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :update_personal_snippet, snippet @@ -120,7 +120,7 @@ module API end delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet @@ -135,7 +135,7 @@ module API end get ":id/raw" do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' @@ -153,7 +153,7 @@ module API snippet = Snippet.find_by!(id: params[:id]) - return not_found!('UserAgentDetail') unless snippet.user_agent_detail + break not_found!('UserAgentDetail') unless snippet.user_agent_detail present snippet.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b3709455bc3..b29e660c6e0 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,7 +62,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger present trigger, with: Entities::Trigger end @@ -99,7 +99,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger if trigger.update(declared_params(include_missing: false)) present trigger, with: Entities::Trigger @@ -119,7 +119,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger if trigger.update(owner: current_user) status :ok @@ -140,7 +140,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger destroy_conditionally!(trigger) end diff --git a/lib/api/users.rb b/lib/api/users.rb index 3920171205f..e8df2c5a74a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -77,7 +77,7 @@ module API authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? - params.except!(:created_after, :created_before, :order_by, :sort) + params.except!(:created_after, :created_before, :order_by, :sort, :two_factor) end users = UsersFinder.new(current_user, params).execute @@ -531,18 +531,22 @@ module API authenticate! end - desc 'Get the currently authenticated user' do - success Entities::UserPublic - end - get do - entity = - if current_user.admin? - Entities::UserWithAdmin - else - Entities::UserPublic - end + # Enabling /user endpoint for the v3 version to allow oauth + # authentication through this endpoint. + version %w(v3 v4), using: :path do + desc 'Get the currently authenticated user' do + success Entities::UserPublic + end + get do + entity = + if current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end - present current_user, with: entity + present current_user, with: entity + end end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/award_emoji.rb b/lib/api/v3/award_emoji.rb deleted file mode 100644 index b96b2d70b12..00000000000 --- a/lib/api/v3/award_emoji.rb +++ /dev/null @@ -1,130 +0,0 @@ -module API - module V3 - class AwardEmoji < Grape::API - include PaginationParams - - before { authenticate! } - AWARDABLES = %w[issue merge_request snippet].freeze - - resource :projects, requirements: { id: %r{[^/]+} } do - AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.pluralize - awardable_id_string = "#{awardable_type}_id" - - params do - requires :id, type: String, desc: 'The ID of a project' - requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" - end - - [ - ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", - ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" - ].each do |endpoint| - - desc 'Get a list of project +awardable+ award emoji' do - detail 'This feature was introduced in 8.9' - success Entities::AwardEmoji - end - params do - use :pagination - end - get endpoint do - if can_read_awardable? - awards = awardable.award_emoji - present paginate(awards), with: Entities::AwardEmoji - else - not_found!("Award Emoji") - end - end - - desc 'Get a specific award emoji' do - detail 'This feature was introduced in 8.9' - success Entities::AwardEmoji - end - params do - requires :award_id, type: Integer, desc: 'The ID of the award' - end - get "#{endpoint}/:award_id" do - if can_read_awardable? - present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji - else - not_found!("Award Emoji") - end - end - - desc 'Award a new Emoji' do - detail 'This feature was introduced in 8.9' - success Entities::AwardEmoji - end - params do - requires :name, type: String, desc: 'The name of a award_emoji (without colons)' - end - post endpoint do - not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? - - award = awardable.create_award_emoji(params[:name], current_user) - - if award.persisted? - present award, with: Entities::AwardEmoji - else - not_found!("Award Emoji #{award.errors.messages}") - end - end - - desc 'Delete a +awardables+ award emoji' do - detail 'This feature was introduced in 8.9' - success Entities::AwardEmoji - end - params do - requires :award_id, type: Integer, desc: 'The ID of an award emoji' - end - delete "#{endpoint}/:award_id" do - award = awardable.award_emoji.find(params[:award_id]) - - unauthorized! unless award.user == current_user || current_user.admin? - - award.destroy - present award, with: Entities::AwardEmoji - end - end - end - end - - helpers do - def can_read_awardable? - can?(current_user, read_ability(awardable), awardable) - end - - def can_award_awardable? - awardable.user_can_award?(current_user, params[:name]) - end - - def awardable - @awardable ||= - begin - if params.include?(:note_id) - note_id = params.delete(:note_id) - - awardable.notes.find(note_id) - elsif params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) - elsif params.include?(:merge_request_id) - user_project.merge_requests.find(params[:merge_request_id]) - else - user_project.snippets.find(params[:snippet_id]) - end - end - end - - def read_ability(awardable) - case awardable - when Note - read_ability(awardable.noteable) - else - :"read_#{awardable.class.to_s.underscore}" - end - end - end - end - end -end diff --git a/lib/api/v3/boards.rb b/lib/api/v3/boards.rb deleted file mode 100644 index 94acc67171e..00000000000 --- a/lib/api/v3/boards.rb +++ /dev/null @@ -1,72 +0,0 @@ -module API - module V3 - class Boards < Grape::API - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get all project boards' do - detail 'This feature was introduced in 8.13' - success ::API::Entities::Board - end - get ':id/boards' do - authorize!(:read_board, user_project) - present user_project.boards, with: ::API::Entities::Board - end - - params do - requires :board_id, type: Integer, desc: 'The ID of a board' - end - segment ':id/boards/:board_id' do - helpers do - def project_board - board = user_project.boards.first - - if params[:board_id] == board.id - board - else - not_found!('Board') - end - end - - def board_lists - project_board.lists.destroyable - end - end - - desc 'Get the lists of a project board' do - detail 'Does not include `done` list. This feature was introduced in 8.13' - success ::API::Entities::List - end - get '/lists' do - authorize!(:read_board, user_project) - present board_lists, with: ::API::Entities::List - end - - desc 'Delete a board list' do - detail 'This feature was introduced in 8.13' - success ::API::Entities::List - end - params do - requires :list_id, type: Integer, desc: 'The ID of a board list' - end - delete "/lists/:list_id" do - authorize!(:admin_list, user_project) - - list = board_lists.find(params[:list_id]) - - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - - if service.execute(list) - present list, with: ::API::Entities::List - else - render_api_error!({ error: 'List could not be deleted!' }, 400) - end - end - end - end - end - end -end diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb deleted file mode 100644 index 25176c5b38e..00000000000 --- a/lib/api/v3/branches.rb +++ /dev/null @@ -1,76 +0,0 @@ -require 'mime/types' - -module API - module V3 - class Branches < Grape::API - before { authenticate! } - before { authorize! :download_code, user_project } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a project repository branches' do - success ::API::Entities::Branch - end - get ":id/repository/branches" do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42276') - - repository = user_project.repository - branches = repository.branches.sort_by(&:name) - merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - - present branches, with: ::API::Entities::Branch, project: user_project, merged_branch_names: merged_branch_names - end - - desc 'Delete a branch' - params do - requires :branch, type: String, desc: 'The name of the branch' - end - delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do - authorize_push_project - - result = DeleteBranchService.new(user_project, current_user) - .execute(params[:branch]) - - if result[:status] == :success - status(200) - { - branch_name: params[:branch] - } - else - render_api_error!(result[:message], result[:return_code]) - end - end - - desc 'Delete all merged branches' - delete ":id/repository/merged_branches" do - DeleteMergedBranchesService.new(user_project, current_user).async_execute - - status(200) - end - - desc 'Create branch' do - success ::API::Entities::Branch - end - params do - requires :branch_name, type: String, desc: 'The name of the branch' - requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' - end - post ":id/repository/branches" do - authorize_push_project - result = CreateBranchService.new(user_project, current_user) - .execute(params[:branch_name], params[:ref]) - - if result[:status] == :success - present result[:branch], - with: ::API::Entities::Branch, - project: user_project - else - render_api_error!(result[:message], 400) - end - end - end - end - end -end diff --git a/lib/api/v3/broadcast_messages.rb b/lib/api/v3/broadcast_messages.rb deleted file mode 100644 index 417e4ad0b26..00000000000 --- a/lib/api/v3/broadcast_messages.rb +++ /dev/null @@ -1,31 +0,0 @@ -module API - module V3 - class BroadcastMessages < Grape::API - include PaginationParams - - before { authenticate! } - before { authenticated_as_admin! } - - resource :broadcast_messages do - helpers do - def find_message - BroadcastMessage.find(params[:id]) - end - end - - desc 'Delete a broadcast message' do - detail 'This feature was introduced in GitLab 8.12.' - success ::API::Entities::BroadcastMessage - end - params do - requires :id, type: Integer, desc: 'Broadcast message ID' - end - delete ':id' do - message = find_message - - present message.destroy, with: ::API::Entities::BroadcastMessage - end - end - end - end -end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb deleted file mode 100644 index ac76fece931..00000000000 --- a/lib/api/v3/builds.rb +++ /dev/null @@ -1,250 +0,0 @@ -module API - module V3 - class Builds < Grape::API - include PaginationParams - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - helpers do - params :optional_scope do - optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', - values: %w(pending running failed success canceled skipped), - coerce_with: ->(scope) { - if scope.is_a?(String) - [scope] - elsif scope.is_a?(::Hash) - scope.values - else - ['unknown'] - end - } - end - end - - desc 'Get a project builds' do - success ::API::V3::Entities::Build - end - params do - use :optional_scope - use :pagination - end - get ':id/builds' do - builds = user_project.builds.order('id DESC') - builds = filter_builds(builds, params[:scope]) - - builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project) - present paginate(builds), with: ::API::V3::Entities::Build - end - - desc 'Get builds for a specific commit of a project' do - success ::API::V3::Entities::Build - end - params do - requires :sha, type: String, desc: 'The SHA id of a commit' - use :optional_scope - use :pagination - end - get ':id/repository/commits/:sha/builds' do - authorize_read_builds! - - return not_found! unless user_project.commit(params[:sha]) - - pipelines = user_project.pipelines.where(sha: params[:sha]) - builds = user_project.builds.where(pipeline: pipelines).order('id DESC') - builds = filter_builds(builds, params[:scope]) - - present paginate(builds), with: ::API::V3::Entities::Build - end - - desc 'Get a specific build of a project' do - success ::API::V3::Entities::Build - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - get ':id/builds/:build_id' do - authorize_read_builds! - - build = get_build!(params[:build_id]) - - present build, with: ::API::V3::Entities::Build - end - - desc 'Download the artifacts file from build' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - get ':id/builds/:build_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:build_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download the artifacts file from build' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the build' - end - get ':id/builds/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_read_builds! - - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) - - present_artifacts!(latest_build.artifacts_file) - end - - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace - # is saved in the DB instead of file). But before that, we need to consider how to replace the value of - # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. - desc 'Get a trace of a specific build of a project' - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - get ':id/builds/:build_id/trace' do - authorize_read_builds! - - build = get_build!(params[:build_id]) - - header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" - content_type 'text/plain' - env['api.format'] = :binary - - trace = build.trace.raw - body trace - end - - desc 'Cancel a specific build of a project' do - success ::API::V3::Entities::Build - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - post ':id/builds/:build_id/cancel' do - authorize_update_builds! - - build = get_build!(params[:build_id]) - authorize!(:update_build, build) - - build.cancel - - present build, with: ::API::V3::Entities::Build - end - - desc 'Retry a specific build of a project' do - success ::API::V3::Entities::Build - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - post ':id/builds/:build_id/retry' do - authorize_update_builds! - - build = get_build!(params[:build_id]) - authorize!(:update_build, build) - return forbidden!('Build is not retryable') unless build.retryable? - - build = Ci::Build.retry(build, current_user) - - present build, with: ::API::V3::Entities::Build - end - - desc 'Erase build (remove artifacts and build trace)' do - success ::API::V3::Entities::Build - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - post ':id/builds/:build_id/erase' do - authorize_update_builds! - - build = get_build!(params[:build_id]) - authorize!(:erase_build, build) - return forbidden!('Build is not erasable!') unless build.erasable? - - build.erase(erased_by: current_user) - present build, with: ::API::V3::Entities::Build - end - - desc 'Keep the artifacts to prevent them from being deleted' do - success ::API::V3::Entities::Build - end - params do - requires :build_id, type: Integer, desc: 'The ID of a build' - end - post ':id/builds/:build_id/artifacts/keep' do - authorize_update_builds! - - build = get_build!(params[:build_id]) - authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: ::API::V3::Entities::Build - end - - desc 'Trigger a manual build' do - success ::API::V3::Entities::Build - detail 'This feature was added in GitLab 8.11' - end - params do - requires :build_id, type: Integer, desc: 'The ID of a Build' - end - post ":id/builds/:build_id/play" do - authorize_read_builds! - - build = get_build!(params[:build_id]) - authorize!(:update_build, build) - bad_request!("Unplayable Job") unless build.playable? - - build.play(current_user) - - status 200 - present build, with: ::API::V3::Entities::Build - end - end - - helpers do - def find_build(id) - user_project.builds.find_by(id: id.to_i) - end - - def get_build!(id) - find_build(id) || not_found! - end - - def filter_builds(builds, scope) - return builds if scope.nil? || scope.empty? - - available_statuses = ::CommitStatus::AVAILABLE_STATUSES - - unknown = scope - available_statuses - render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? - - builds.where(status: available_statuses && scope) - end - - def authorize_read_builds! - authorize! :read_build, user_project - end - - def authorize_update_builds! - authorize! :update_build, user_project - end - end - end - end -end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb deleted file mode 100644 index 4f6ea8f502e..00000000000 --- a/lib/api/v3/commits.rb +++ /dev/null @@ -1,199 +0,0 @@ -require 'mime/types' - -module API - module V3 - class Commits < Grape::API - include PaginationParams - - before { authenticate! } - before { authorize! :download_code, user_project } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Get a project repository commits' do - success ::API::Entities::Commit - end - params do - optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :since, type: DateTime, desc: 'Only commits after or in this date will be returned' - optional :until, type: DateTime, desc: 'Only commits before or in this date will be returned' - optional :page, type: Integer, default: 0, desc: 'The page for pagination' - optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' - optional :path, type: String, desc: 'The file path' - end - get ":id/repository/commits" do - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - offset = params[:page] * params[:per_page] - - commits = user_project.repository.commits(ref, - path: params[:path], - limit: params[:per_page], - offset: offset, - after: params[:since], - before: params[:until]) - - present commits, with: ::API::Entities::Commit - end - - desc 'Commit multiple file changes as one commit' do - success ::API::Entities::CommitDetail - detail 'This feature was introduced in GitLab 8.13' - end - params do - requires :branch_name, type: String, desc: 'The name of branch' - requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' - optional :author_email, type: String, desc: 'Author email for commit' - optional :author_name, type: String, desc: 'Author name for commit' - end - post ":id/repository/commits" do - authorize! :push_code, user_project - - attrs = declared_params.dup - branch = attrs.delete(:branch_name) - attrs.merge!(start_branch: branch, branch_name: branch) - - result = ::Files::MultiService.new(user_project, current_user, attrs).execute - - if result[:status] == :success - commit_detail = user_project.repository.commits(result[:result], limit: 1).first - present commit_detail, with: ::API::Entities::CommitDetail - else - render_api_error!(result[:message], 400) - end - end - - desc 'Get a specific commit of a project' do - success ::API::Entities::CommitDetail - failure [[404, 'Not Found']] - end - params do - requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' - optional :stats, type: Boolean, default: true, desc: 'Include commit stats' - end - get ":id/repository/commits/:sha", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - commit = user_project.commit(params[:sha]) - - not_found! "Commit" unless commit - - present commit, with: ::API::Entities::CommitDetail, stats: params[:stats] - end - - desc 'Get the diff for a specific commit of a project' do - failure [[404, 'Not Found']] - end - params do - requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' - end - get ":id/repository/commits/:sha/diff", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - commit = user_project.commit(params[:sha]) - - not_found! "Commit" unless commit - - commit.raw_diffs.to_a - end - - desc "Get a commit's comments" do - success ::API::Entities::CommitNote - failure [[404, 'Not Found']] - end - params do - use :pagination - requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' - end - get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - commit = user_project.commit(params[:sha]) - - not_found! 'Commit' unless commit - notes = commit.notes.order(:created_at) - - present paginate(notes), with: ::API::Entities::CommitNote - end - - desc 'Cherry pick commit into a branch' do - detail 'This feature was introduced in GitLab 8.15' - success ::API::Entities::Commit - end - params do - requires :sha, type: String, desc: 'A commit sha to be cherry picked' - requires :branch, type: String, desc: 'The name of the branch' - end - post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - authorize! :push_code, user_project - - commit = user_project.commit(params[:sha]) - not_found!('Commit') unless commit - - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch - - commit_params = { - commit: commit, - start_branch: params[:branch], - branch_name: params[:branch] - } - - result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute - - if result[:status] == :success - branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::Commit - else - render_api_error!(result[:message], 400) - end - end - - desc 'Post comment to commit' do - success ::API::Entities::CommitNote - end - params do - requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" - requires :note, type: String, desc: 'The text of the comment' - optional :path, type: String, desc: 'The file path' - given :path do - requires :line, type: Integer, desc: 'The line number' - requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' - end - end - post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - commit = user_project.commit(params[:sha]) - not_found! 'Commit' unless commit - - opts = { - note: params[:note], - noteable_type: 'Commit', - commit_id: commit.id - } - - if params[:path] - commit.raw_diffs(limits: false).each do |diff| - next unless diff.new_path == params[:path] - - lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) - - lines.each do |line| - next unless line.new_pos == params[:line] && line.type == params[:line_type] - - break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) - end - - break if opts[:line_code] - end - - opts[:type] = LegacyDiffNote.name if opts[:line_code] - end - - note = ::Notes::CreateService.new(user_project, current_user, opts).execute - - if note.save - present note, with: ::API::Entities::CommitNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end - end - end - end - end -end diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb deleted file mode 100644 index 47e54ca85a5..00000000000 --- a/lib/api/v3/deploy_keys.rb +++ /dev/null @@ -1,143 +0,0 @@ -module API - module V3 - class DeployKeys < Grape::API - before { authenticate! } - - helpers do - def add_deploy_keys_project(project, attrs = {}) - project.deploy_keys_projects.create(attrs) - end - - def find_by_deploy_key(project, key_id) - project.deploy_keys_projects.find_by!(deploy_key: key_id) - end - end - - get "deploy_keys" do - authenticated_as_admin! - - keys = DeployKey.all - present keys, with: ::API::Entities::SSHKey - end - - params do - requires :id, type: String, desc: 'The ID of the project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - before { authorize_admin_project } - - %w(keys deploy_keys).each do |path| - desc "Get a specific project's deploy keys" do - success ::API::Entities::DeployKeysProject - end - get ":id/#{path}" do - keys = user_project.deploy_keys_projects.preload(:deploy_key) - - present keys, with: ::API::Entities::DeployKeysProject - end - - desc 'Get single deploy key' do - success ::API::Entities::DeployKeysProject - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - get ":id/#{path}/:key_id" do - key = find_by_deploy_key(user_project, params[:key_id]) - - present key, with: ::API::Entities::DeployKeysProject - end - - desc 'Add new deploy key to currently authenticated user' do - success ::API::Entities::DeployKeysProject - end - params do - requires :key, type: String, desc: 'The new deploy key' - requires :title, type: String, desc: 'The name of the deploy key' - optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" - end - post ":id/#{path}" do - params[:key].strip! - - # Check for an existing key joined to this project - key = user_project.deploy_keys_projects - .joins(:deploy_key) - .find_by(keys: { key: params[:key] }) - - if key - present key, with: ::API::Entities::DeployKeysProject - break - end - - # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by(key: params[:key]) - if key - added_key = add_deploy_keys_project(user_project, deploy_key: key, can_push: !!params[:can_push]) - - present added_key, with: ::API::Entities::DeployKeysProject - break - end - - # Create a new deploy key - key_attributes = { can_push: !!params[:can_push], - deploy_key_attributes: declared_params.except(:can_push) } - key = add_deploy_keys_project(user_project, key_attributes) - - if key.valid? - present key, with: ::API::Entities::DeployKeysProject - else - render_validation_error!(key) - end - end - - desc 'Enable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success ::API::Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - post ":id/#{path}/:key_id/enable" do - key = ::Projects::EnableDeployKeyService.new(user_project, - current_user, declared_params).execute - - if key - present key, with: ::API::Entities::SSHKey - else - not_found!('Deploy Key') - end - end - - desc 'Disable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success ::API::Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id/disable" do - key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) - key.destroy - - present key.deploy_key, with: ::API::Entities::SSHKey - end - - desc 'Delete deploy key for a project' do - success Key - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id" do - key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) - if key - key.destroy - else - not_found!('Deploy Key') - end - end - end - end - end - end -end diff --git a/lib/api/v3/deployments.rb b/lib/api/v3/deployments.rb deleted file mode 100644 index 1d4972eda26..00000000000 --- a/lib/api/v3/deployments.rb +++ /dev/null @@ -1,43 +0,0 @@ -module API - module V3 - # Deployments RESTful API endpoints - class Deployments < Grape::API - include PaginationParams - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The project ID' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get all deployments of the project' do - detail 'This feature was introduced in GitLab 8.11.' - success ::API::V3::Deployments - end - params do - use :pagination - end - get ':id/deployments' do - authorize! :read_deployment, user_project - - present paginate(user_project.deployments), with: ::API::V3::Deployments - end - - desc 'Gets a specific deployment' do - detail 'This feature was introduced in GitLab 8.11.' - success ::API::V3::Deployments - end - params do - requires :deployment_id, type: Integer, desc: 'The deployment ID' - end - get ':id/deployments/:deployment_id' do - authorize! :read_deployment, user_project - - deployment = user_project.deployments.find(params[:deployment_id]) - - present deployment, with: ::API::V3::Deployments - end - end - end - end -end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb deleted file mode 100644 index 68b4d7c3982..00000000000 --- a/lib/api/v3/entities.rb +++ /dev/null @@ -1,309 +0,0 @@ -module API - module V3 - module Entities - class ProjectSnippet < Grape::Entity - expose :id, :title, :file_name - expose :author, using: ::API::Entities::UserBasic - expose :updated_at, :created_at - expose(:expires_at) { |snippet| nil } - - expose :web_url do |snippet, options| - Gitlab::UrlBuilder.build(snippet) - end - end - - class Note < Grape::Entity - expose :id - expose :note, as: :body - expose :attachment_identifier, as: :attachment - expose :author, using: ::API::Entities::UserBasic - expose :created_at, :updated_at - expose :system?, as: :system - expose :noteable_id, :noteable_type - # upvote? and downvote? are deprecated, always return false - expose(:upvote?) { |note| false } - expose(:downvote?) { |note| false } - end - - class PushEventPayload < Grape::Entity - expose :commit_count, :action, :ref_type, :commit_from, :commit_to - expose :ref, :commit_title - end - - class Event < Grape::Entity - expose :project_id, :action_name - expose :target_id, :target_type, :author_id - expose :target_title - expose :created_at - expose :note, using: Entities::Note, if: ->(event, options) { event.note? } - expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author } - - expose :push_event_payload, - as: :push_data, - using: PushEventPayload, - if: -> (event, _) { event.push? } - - expose :author_username do |event, options| - event.author&.username - end - end - - class AwardEmoji < Grape::Entity - expose :id - expose :name - expose :user, using: ::API::Entities::UserBasic - expose :created_at, :updated_at - expose :awardable_id, :awardable_type - end - - class Project < Grape::Entity - expose :id, :description, :default_branch, :tag_list - expose :public?, as: :public - expose :archived?, as: :archived - expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url - expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } - expose :name, :name_with_namespace - expose :path, :path_with_namespace - expose :resolve_outdated_diff_discussions - expose :container_registry_enabled - - # Expose old field names with the new permissions methods to keep API compatible - expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } - expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } - expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } - expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } - expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } - - expose :created_at, :last_activity_at - expose :shared_runners_enabled - expose :lfs_enabled?, as: :lfs_enabled - expose :creator_id - expose :namespace, using: 'API::Entities::Namespace' - expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end - expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } - expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :public_builds - expose :shared_with_groups do |project, options| - ::API::Entities::SharedGroup.represent(project.project_group_links.all, options) - end - expose :only_allow_merge_if_pipeline_succeeds, as: :only_allow_merge_if_build_succeeds - expose :request_access_enabled - expose :only_allow_merge_if_all_discussions_are_resolved - - expose :statistics, using: '::API::V3::Entities::ProjectStatistics', if: :statistics - end - - class ProjectWithAccess < Project - expose :permissions do - expose :project_access, using: ::API::Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:current_user].id) - end - - expose :group_access, using: ::API::Entities::GroupAccess do |project, options| - if project.group - project.group.group_members.find_by(user_id: options[:current_user].id) - end - end - end - end - - class MergeRequest < Grape::Entity - expose :id, :iid - expose(:project_id) { |entity| entity.project.id } - expose :title, :description - expose :state, :created_at, :updated_at - expose :target_branch, :source_branch - expose :upvotes, :downvotes - expose :author, :assignee, using: ::API::Entities::UserBasic - expose :source_project_id, :target_project_id - expose :label_names, as: :labels - expose :work_in_progress?, as: :work_in_progress - expose :milestone, using: ::API::Entities::Milestone - expose :merge_when_pipeline_succeeds, as: :merge_when_build_succeeds - expose :merge_status - expose :diff_head_sha, as: :sha - expose :merge_commit_sha - expose :subscribed do |merge_request, options| - merge_request.subscribed?(options[:current_user], options[:project]) - end - expose :user_notes_count - expose :should_remove_source_branch?, as: :should_remove_source_branch - expose :force_remove_source_branch?, as: :force_remove_source_branch - - expose :web_url do |merge_request, options| - Gitlab::UrlBuilder.build(merge_request) - end - end - - class Group < Grape::Entity - expose :id, :name, :path, :description, :visibility_level - expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end - expose :web_url - expose :request_access_enabled - expose :full_name, :full_path - - if ::Group.supports_nested_groups? - expose :parent_id - end - - expose :statistics, if: :statistics do - with_options format_with: -> (value) { value.to_i } do - expose :storage_size - expose :repository_size - expose :lfs_objects_size - expose :build_artifacts_size - end - end - end - - class GroupDetail < Group - expose :projects, using: Entities::Project - expose :shared_projects, using: Entities::Project - end - - class ApplicationSetting < Grape::Entity - expose :id - expose :default_projects_limit - expose :signup_enabled - expose :password_authentication_enabled_for_web, as: :password_authentication_enabled - expose :password_authentication_enabled_for_web, as: :signin_enabled - expose :gravatar_enabled - expose :sign_in_text - expose :after_sign_up_text - expose :created_at - expose :updated_at - expose :home_page_url - expose :default_branch_protection - expose :restricted_visibility_levels - expose :max_attachment_size - expose :session_expire_delay - expose :default_project_visibility - expose :default_snippet_visibility - expose :default_group_visibility - expose :domain_whitelist - expose :domain_blacklist_enabled - expose :domain_blacklist - expose :user_oauth_applications - expose :after_sign_out_path - expose :container_registry_token_expire_delay - expose :repository_storage - expose :repository_storages - expose :koding_enabled - expose :koding_url - expose :plantuml_enabled - expose :plantuml_url - expose :terminal_max_session_time - end - - class Environment < ::API::Entities::EnvironmentBasic - expose :project, using: Entities::Project - end - - class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :last_used - expose :owner, using: ::API::Entities::UserBasic - end - - class TriggerRequest < Grape::Entity - expose :id, :variables - end - - class Build < Grape::Entity - expose :id, :status, :stage, :name, :ref, :tag, :coverage - expose :created_at, :started_at, :finished_at - expose :user, with: ::API::Entities::User - expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: ::API::Entities::Commit - expose :runner, with: ::API::Entities::Runner - expose :pipeline, with: ::API::Entities::PipelineBasic - end - - class BuildArtifactFile < Grape::Entity - expose :filename, :size - end - - class Deployment < Grape::Entity - expose :id, :iid, :ref, :sha, :created_at - expose :user, using: ::API::Entities::UserBasic - expose :environment, using: ::API::Entities::EnvironmentBasic - expose :deployable, using: Entities::Build - end - - class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: ::API::Entities::Diff do |compare, _| - compare.raw_diffs(limits: false).to_a - end - end - - class ProjectStatistics < Grape::Entity - expose :commit_count - expose :storage_size - expose :repository_size - expose :lfs_objects_size - expose :build_artifacts_size - end - - class ProjectService < Grape::Entity - expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :confidential_issues_events - expose :merge_requests_events, :tag_push_events, :note_events - expose :pipeline_events - expose :job_events, as: :build_events - # Expose serialized properties - expose :properties do |service, options| - service.properties.slice(*service.api_field_names) - end - end - - class ProjectHook < ::API::Entities::Hook - expose :project_id, :issues_events, :confidential_issues_events - expose :merge_requests_events, :note_events, :pipeline_events - expose :wiki_page_events - expose :job_events, as: :build_events - end - - class ProjectEntity < Grape::Entity - expose :id, :iid - expose(:project_id) { |entity| entity&.project.try(:id) } - expose :title, :description - expose :state, :created_at, :updated_at - end - - class IssueBasic < ProjectEntity - expose :label_names, as: :labels - expose :milestone, using: ::API::Entities::Milestone - expose :assignees, :author, using: ::API::Entities::UserBasic - - expose :assignee, using: ::API::Entities::UserBasic do |issue, options| - issue.assignees.first - end - - expose :user_notes_count - expose :upvotes, :downvotes - expose :due_date - expose :confidential - - expose :web_url do |issue, options| - Gitlab::UrlBuilder.build(issue) - end - end - - class Issue < IssueBasic - unexpose :assignees - expose :assignee do |issue, options| - ::API::Entities::UserBasic.represent(issue.assignees.first, options) - end - expose :subscribed do |issue, options| - issue.subscribed?(options[:current_user], options[:project] || issue.project) - end - end - end - end -end diff --git a/lib/api/v3/environments.rb b/lib/api/v3/environments.rb deleted file mode 100644 index 6bb4e016a01..00000000000 --- a/lib/api/v3/environments.rb +++ /dev/null @@ -1,87 +0,0 @@ -module API - module V3 - class Environments < Grape::API - include ::API::Helpers::CustomValidators - include PaginationParams - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The project ID' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get all environments of the project' do - detail 'This feature was introduced in GitLab 8.11.' - success Entities::Environment - end - params do - use :pagination - end - get ':id/environments' do - authorize! :read_environment, user_project - - present paginate(user_project.environments), with: Entities::Environment - end - - desc 'Creates a new environment' do - detail 'This feature was introduced in GitLab 8.11.' - success Entities::Environment - end - params do - requires :name, type: String, desc: 'The name of the environment to be created' - optional :external_url, type: String, desc: 'URL on which this deployment is viewable' - optional :slug, absence: { message: "is automatically generated and cannot be changed" } - end - post ':id/environments' do - authorize! :create_environment, user_project - - environment = user_project.environments.create(declared_params) - - if environment.persisted? - present environment, with: Entities::Environment - else - render_validation_error!(environment) - end - end - - desc 'Updates an existing environment' do - detail 'This feature was introduced in GitLab 8.11.' - success Entities::Environment - end - params do - requires :environment_id, type: Integer, desc: 'The environment ID' - optional :name, type: String, desc: 'The new environment name' - optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' - optional :slug, absence: { message: "is automatically generated and cannot be changed" } - end - put ':id/environments/:environment_id' do - authorize! :update_environment, user_project - - environment = user_project.environments.find(params[:environment_id]) - - update_params = declared_params(include_missing: false).extract!(:name, :external_url) - if environment.update(update_params) - present environment, with: Entities::Environment - else - render_validation_error!(environment) - end - end - - desc 'Deletes an existing environment' do - detail 'This feature was introduced in GitLab 8.11.' - success Entities::Environment - end - params do - requires :environment_id, type: Integer, desc: 'The environment ID' - end - delete ':id/environments/:environment_id' do - authorize! :update_environment, user_project - - environment = user_project.environments.find(params[:environment_id]) - - present environment.destroy, with: Entities::Environment - end - end - end - end -end diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb deleted file mode 100644 index 7b4b3448b6d..00000000000 --- a/lib/api/v3/files.rb +++ /dev/null @@ -1,138 +0,0 @@ -module API - module V3 - class Files < Grape::API - helpers do - def commit_params(attrs) - { - file_path: attrs[:file_path], - start_branch: attrs[:branch], - branch_name: attrs[:branch], - commit_message: attrs[:commit_message], - file_content: attrs[:content], - file_content_encoding: attrs[:encoding], - author_email: attrs[:author_email], - author_name: attrs[:author_name] - } - end - - def commit_response(attrs) - { - file_path: attrs[:file_path], - branch: attrs[:branch] - } - end - - params :simple_file_params do - requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' - requires :branch_name, type: String, desc: 'The name of branch' - requires :commit_message, type: String, desc: 'Commit Message' - optional :author_email, type: String, desc: 'The email of the author' - optional :author_name, type: String, desc: 'The name of the author' - end - - params :extended_file_params do - use :simple_file_params - requires :content, type: String, desc: 'File content' - optional :encoding, type: String, values: %w[base64], desc: 'File encoding' - end - end - - params do - requires :id, type: String, desc: 'The project ID' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a file from repository' - params do - requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' - requires :ref, type: String, desc: 'The name of branch, tag, or commit' - end - get ":id/repository/files" do - authorize! :download_code, user_project - - commit = user_project.commit(params[:ref]) - not_found!('Commit') unless commit - - repo = user_project.repository - blob = repo.blob_at(commit.sha, params[:file_path]) - not_found!('File') unless blob - - blob.load_all_data! - status(200) - - { - file_name: blob.name, - file_path: blob.path, - size: blob.size, - encoding: "base64", - content: Base64.strict_encode64(blob.data), - ref: params[:ref], - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) - } - end - - desc 'Create new file in repository' - params do - use :extended_file_params - end - post ":id/repository/files" do - authorize! :push_code, user_project - - file_params = declared_params(include_missing: false) - file_params[:branch] = file_params.delete(:branch_name) - - result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute - - if result[:status] == :success - status(201) - commit_response(file_params) - else - render_api_error!(result[:message], 400) - end - end - - desc 'Update existing file in repository' - params do - use :extended_file_params - end - put ":id/repository/files" do - authorize! :push_code, user_project - - file_params = declared_params(include_missing: false) - file_params[:branch] = file_params.delete(:branch_name) - - result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute - - if result[:status] == :success - status(200) - commit_response(file_params) - else - http_status = result[:http_status] || 400 - render_api_error!(result[:message], http_status) - end - end - - desc 'Delete an existing file in repository' - params do - use :simple_file_params - end - delete ":id/repository/files" do - authorize! :push_code, user_project - - file_params = declared_params(include_missing: false) - file_params[:branch] = file_params.delete(:branch_name) - - result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute - - if result[:status] == :success - status(200) - commit_response(file_params) - else - render_api_error!(result[:message], 400) - end - end - end - end - end -end diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb deleted file mode 100644 index 2c52d21fa1c..00000000000 --- a/lib/api/v3/groups.rb +++ /dev/null @@ -1,185 +0,0 @@ -module API - module V3 - class Groups < Grape::API - include PaginationParams - - before { authenticate! } - - helpers do - params :optional_params do - optional :description, type: String, desc: 'The description of the group' - optional :visibility_level, type: Integer, desc: 'The visibility level of the group' - optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - end - - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - - def present_groups(groups, options = {}) - options = options.reverse_merge( - with: Entities::Group, - current_user: current_user - ) - - groups = groups.with_statistics if options[:statistics] - present paginate(groups), options - end - end - - resource :groups do - desc 'Get a groups list' do - success Entities::Group - end - params do - use :statistics_params - optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' - optional :all_available, type: Boolean, desc: 'Show all group that you have access to' - optional :search, type: String, desc: 'Search for a specific group' - optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' - optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' - use :pagination - end - get do - groups = if current_user.admin - Group.all - elsif params[:all_available] - GroupsFinder.new(current_user).execute - else - current_user.groups - end - - groups = groups.search(params[:search]) if params[:search].present? - groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - groups = groups.reorder(params[:order_by] => params[:sort]) - - present_groups groups, statistics: params[:statistics] && current_user.admin? - end - - desc 'Get list of owned groups for authenticated user' do - success Entities::Group - end - params do - use :pagination - use :statistics_params - end - get '/owned' do - present_groups current_user.owned_groups, statistics: params[:statistics] - end - - desc 'Create a group. Available only for users who can create groups.' do - success Entities::Group - end - params do - requires :name, type: String, desc: 'The name of the group' - requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_groups? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end - - use :optional_params - end - post do - authorize! :create_group - - group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute - - if group.persisted? - present group, with: Entities::Group, current_user: current_user - else - render_api_error!("Failed to save group #{group.errors.messages}", 400) - end - end - end - - params do - requires :id, type: String, desc: 'The ID of a group' - end - resource :groups, requirements: { id: %r{[^/]+} } do - desc 'Update a group. Available only for users who can administrate groups.' do - success Entities::Group - end - params do - optional :name, type: String, desc: 'The name of the group' - optional :path, type: String, desc: 'The path of the group' - use :optional_params - at_least_one_of :name, :path, :description, :visibility_level, - :lfs_enabled, :request_access_enabled - end - put ':id' do - group = find_group!(params[:id]) - authorize! :admin_group, group - - if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute - present group, with: Entities::GroupDetail, current_user: current_user - else - render_validation_error!(group) - end - end - - desc 'Get a single group, with containing projects.' do - success Entities::GroupDetail - end - get ":id" do - group = find_group!(params[:id]) - present group, with: Entities::GroupDetail, current_user: current_user - end - - desc 'Remove a group.' - delete ":id" do - group = find_group!(params[:id]) - authorize! :admin_group, group - present ::Groups::DestroyService.new(group, current_user).execute, with: Entities::GroupDetail, current_user: current_user - end - - desc 'Get a list of projects in this group.' do - success Entities::Project - end - params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' - optional :visibility, type: String, values: %w[public internal private], - desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' - optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], - default: 'created_at', desc: 'Return projects ordered by field' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return projects sorted in ascending and descending order' - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' - - use :pagination - end - get ":id/projects" do - group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute - projects = filter_projects(projects) - entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project - present paginate(projects), with: entity, current_user: current_user - end - - desc 'Transfer a project to the group namespace. Available only for admin.' do - success Entities::GroupDetail - end - params do - requires :project_id, type: String, desc: 'The ID or path of the project' - end - post ":id/projects/:project_id", requirements: { project_id: /.+/ } do - authenticated_as_admin! - group = find_group!(params[:id]) - project = find_project!(params[:project_id]) - result = ::Projects::TransferService.new(project, current_user).execute(group) - - if result - present group, with: Entities::GroupDetail, current_user: current_user - else - render_api_error!("Failed to transfer project #{project.errors.messages}", 400) - end - end - end - end - end -end diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb deleted file mode 100644 index 4e63aa01c1a..00000000000 --- a/lib/api/v3/helpers.rb +++ /dev/null @@ -1,49 +0,0 @@ -module API - module V3 - module Helpers - def find_project_issue(id) - IssuesFinder.new(current_user, project_id: user_project.id).find(id) - end - - def find_project_merge_request(id) - MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) - end - - def find_merge_request_with_access(id, access_level = :read_merge_request) - merge_request = user_project.merge_requests.find(id) - authorize! access_level, merge_request - merge_request - end - - # project helpers - - def filter_projects(projects) - if params[:membership] - projects = projects.merge(current_user.authorized_projects) - end - - if params[:owned] - projects = projects.merge(current_user.owned_projects) - end - - if params[:starred] - projects = projects.merge(current_user.starred_projects) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) - end - - unless params[:archived].nil? - projects = projects.where(archived: to_boolean(params[:archived])) - end - - projects.reorder(params[:order_by] => params[:sort]) - end - end - end -end diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb deleted file mode 100644 index b59947d81d9..00000000000 --- a/lib/api/v3/issues.rb +++ /dev/null @@ -1,240 +0,0 @@ -module API - module V3 - class Issues < Grape::API - include PaginationParams - - before { authenticate! } - - helpers do - def find_issues(args = {}) - args = params.merge(args) - args = convert_parameters_from_legacy_format(args) - - args.delete(:id) - args[:milestone_title] = args.delete(:milestone) - - match_all_labels = args.delete(:match_all_labels) - labels = args.delete(:labels) - args[:label_name] = labels if match_all_labels - - # IssuesFinder expects iids - args[:iids] = args.delete(:iid) if args.key?(:iid) - - issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations - - if !match_all_labels && labels.present? - issues = issues.includes(:labels).where('labels.title' => labels.split(',')) - end - - issues.reorder(args[:order_by] => args[:sort]) - end - - params :issues_params do - optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :milestone, type: String, desc: 'Milestone title' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return issues ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return issues sorted in `asc` or `desc` order.' - optional :milestone, type: String, desc: 'Return issues for a specific milestone' - use :pagination - end - - params :issue_params do - optional :description, type: String, desc: 'The description of an issue' - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' - optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' - optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' - optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' - end - end - - resource :issues do - desc "Get currently authenticated user's issues" do - success ::API::V3::Entities::Issue - end - params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' - use :issues_params - end - get do - issues = find_issues(scope: 'authored') - - present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user - end - end - - params do - requires :id, type: String, desc: 'The ID of a group' - end - resource :groups, requirements: { id: %r{[^/]+} } do - desc 'Get a list of group issues' do - success ::API::V3::Entities::Issue - end - params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' - use :issues_params - end - get ":id/issues" do - group = find_group!(params[:id]) - - issues = find_issues(group_id: group.id, match_all_labels: true) - - present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - include TimeTrackingEndpoints - - desc 'Get a list of project issues' do - detail 'iid filter is deprecated have been removed on V4' - success ::API::V3::Entities::Issue - end - params do - optional :state, type: String, values: %w[opened closed all], default: 'all', - desc: 'Return opened, closed, or all issues' - optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' - use :issues_params - end - get ":id/issues" do - project = find_project!(params[:id]) - - issues = find_issues(project_id: project.id) - - present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - end - - desc 'Get a single project issue' do - success ::API::V3::Entities::Issue - end - params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' - end - get ":id/issues/:issue_id" do - issue = find_project_issue(params[:issue_id]) - present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - end - - desc 'Create a new project issue' do - success ::API::V3::Entities::Issue - end - params do - requires :title, type: String, desc: 'The title of an issue' - optional :created_at, type: DateTime, - desc: 'Date time when the issue was created. Available only for admins and project owners.' - optional :merge_request_for_resolving_discussions, type: Integer, - desc: 'The IID of a merge request for which to resolve discussions' - use :issue_params - end - post ':id/issues' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42131') - - # Setting created_at time only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user - params.delete(:created_at) - end - - issue_params = declared_params(include_missing: false) - issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions)) - issue_params = convert_parameters_from_legacy_format(issue_params) - - issue = ::Issues::CreateService.new(user_project, - current_user, - issue_params.merge(request: request, api: true)).execute - render_spam_error! if issue.spam? - - if issue.valid? - present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - else - render_validation_error!(issue) - end - end - - desc 'Update an existing issue' do - success ::API::V3::Entities::Issue - end - params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' - optional :title, type: String, desc: 'The title of an issue' - optional :updated_at, type: DateTime, - desc: 'Date time when the issue was updated. Available only for admins and project owners.' - optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' - use :issue_params - at_least_one_of :title, :description, :assignee_id, :milestone_id, - :labels, :created_at, :due_date, :confidential, :state_event - end - put ':id/issues/:issue_id' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42132') - - issue = user_project.issues.find(params.delete(:issue_id)) - authorize! :update_issue, issue - - # Setting created_at time only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user - params.delete(:updated_at) - end - - update_params = declared_params(include_missing: false).merge(request: request, api: true) - update_params = convert_parameters_from_legacy_format(update_params) - - issue = ::Issues::UpdateService.new(user_project, - current_user, - update_params).execute(issue) - - render_spam_error! if issue.spam? - - if issue.valid? - present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - else - render_validation_error!(issue) - end - end - - desc 'Move an existing issue' do - success ::API::V3::Entities::Issue - end - params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' - requires :to_project_id, type: Integer, desc: 'The ID of the new project' - end - post ':id/issues/:issue_id/move' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42133') - - issue = user_project.issues.find_by(id: params[:issue_id]) - not_found!('Issue') unless issue - - new_project = Project.find_by(id: params[:to_project_id]) - not_found!('Project') unless new_project - - begin - issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - rescue ::Issues::MoveService::MoveError => error - render_api_error!(error.message, 400) - end - end - - desc 'Delete a project issue' - params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' - end - delete ":id/issues/:issue_id" do - issue = user_project.issues.find_by(id: params[:issue_id]) - not_found!('Issue') unless issue - - authorize!(:destroy_issue, issue) - - status(200) - issue.destroy - end - end - end - end -end diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb deleted file mode 100644 index 4157462ec2a..00000000000 --- a/lib/api/v3/labels.rb +++ /dev/null @@ -1,34 +0,0 @@ -module API - module V3 - class Labels < Grape::API - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get all labels of the project' do - success ::API::Entities::Label - end - get ':id/labels' do - present available_labels_for(user_project), with: ::API::Entities::Label, current_user: current_user, project: user_project - end - - desc 'Delete an existing label' do - success ::API::Entities::Label - end - params do - requires :name, type: String, desc: 'The name of the label to be deleted' - end - delete ':id/labels' do - authorize! :admin_label, user_project - - label = user_project.labels.find_by(title: params[:name]) - not_found!('Label') unless label - - present label.destroy, with: ::API::Entities::Label, current_user: current_user, project: user_project - end - end - end - end -end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb deleted file mode 100644 index 88dd598f1e9..00000000000 --- a/lib/api/v3/members.rb +++ /dev/null @@ -1,136 +0,0 @@ -module API - module V3 - class Members < Grape::API - include PaginationParams - - before { authenticate! } - - helpers ::API::Helpers::MembersHelpers - - %w[group project].each do |source_type| - params do - requires :id, type: String, desc: "The #{source_type} ID" - end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do - desc 'Gets a list of group or project members viewable by the authenticated user.' do - success ::API::Entities::Member - end - params do - optional :query, type: String, desc: 'A query string to search for members' - use :pagination - end - get ":id/members" do - source = find_source(source_type, params[:id]) - - members = source.members.where.not(user_id: nil).includes(:user) - members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present? - members = paginate(members) - - present members, with: ::API::Entities::Member - end - - desc 'Gets a member of a group or project.' do - success ::API::Entities::Member - end - params do - requires :user_id, type: Integer, desc: 'The user ID of the member' - end - get ":id/members/:user_id" do - source = find_source(source_type, params[:id]) - - members = source.members - member = members.find_by!(user_id: params[:user_id]) - - present member, with: ::API::Entities::Member - end - - desc 'Adds a member to a group or project.' do - success ::API::Entities::Member - end - params do - requires :user_id, type: Integer, desc: 'The user ID of the new member' - requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' - optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' - end - post ":id/members" do - source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - - member = source.members.find_by(user_id: params[:user_id]) - - # We need this explicit check because `source.add_user` doesn't - # currently return the member created so it would return 201 even if - # the member already existed... - # The `source_type == 'group'` check is to ensure back-compatibility - # but 409 behavior should be used for both project and group members in 9.0! - conflict!('Member already exists') if source_type == 'group' && member - - unless member - member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - end - - if member.persisted? && member.valid? - present member, with: ::API::Entities::Member - else - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) - render_validation_error!(member) - end - end - - desc 'Updates a member of a group or project.' do - success ::API::Entities::Member - end - params do - requires :user_id, type: Integer, desc: 'The user ID of the new member' - requires :access_level, type: Integer, desc: 'A valid access level' - optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' - end - put ":id/members/:user_id" do - source = find_source(source_type, params.delete(:id)) - authorize_admin_source!(source_type, source) - - member = source.members.find_by!(user_id: params.delete(:user_id)) - - if member.update_attributes(declared_params(include_missing: false)) - present member, with: ::API::Entities::Member - else - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) - render_validation_error!(member) - end - end - - desc 'Removes a user from a group or project.' - params do - requires :user_id, type: Integer, desc: 'The user ID of the member' - end - delete ":id/members/:user_id" do - source = find_source(source_type, params[:id]) - - # This is to ensure back-compatibility but find_by! should be used - # in that casse in 9.0! - member = source.members.find_by(user_id: params[:user_id]) - - # This is to ensure back-compatibility but this should be removed in - # favor of find_by! in 9.0! - not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? - - # This is to ensure back-compatibility but 204 behavior should be used - # for all DELETE endpoints in 9.0! - if member.nil? - status(200 ) - { message: "Access revoked", id: params[:user_id].to_i } - else - ::Members::DestroyService.new(current_user).execute(member) - - present member, with: ::API::Entities::Member - end - end - end - end - end - end -end diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb deleted file mode 100644 index 22866fc2845..00000000000 --- a/lib/api/v3/merge_request_diffs.rb +++ /dev/null @@ -1,44 +0,0 @@ -module API - module V3 - # MergeRequestDiff API - class MergeRequestDiffs < Grape::API - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a list of merge request diff versions' do - detail 'This feature was introduced in GitLab 8.12.' - success ::API::Entities::MergeRequestDiff - end - - params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' - end - - get ":id/merge_requests/:merge_request_id/versions" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - - present merge_request.merge_request_diffs.order_id_desc, with: ::API::Entities::MergeRequestDiff - end - - desc 'Get a single merge request diff version' do - detail 'This feature was introduced in GitLab 8.12.' - success ::API::Entities::MergeRequestDiffFull - end - - params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' - requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' - end - - get ":id/merge_requests/:merge_request_id/versions/:version_id" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - - present merge_request.merge_request_diffs.find(params[:version_id]), with: ::API::Entities::MergeRequestDiffFull - end - end - end - end -end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb deleted file mode 100644 index ce216497996..00000000000 --- a/lib/api/v3/merge_requests.rb +++ /dev/null @@ -1,297 +0,0 @@ -module API - module V3 - class MergeRequests < Grape::API - include PaginationParams - - DEPRECATION_MESSAGE = 'This endpoint is deprecated and has been removed on V4'.freeze - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - include TimeTrackingEndpoints - - helpers do - def handle_merge_request_errors!(errors) - if errors[:project_access].any? - error!(errors[:project_access], 422) - elsif errors[:branch_conflict].any? - error!(errors[:branch_conflict], 422) - elsif errors[:validate_fork].any? - error!(errors[:validate_fork], 422) - elsif errors[:validate_branches].any? - conflict!(errors[:validate_branches]) - elsif errors[:base].any? - error!(errors[:base], 422) - end - - render_api_error!(errors, 400) - end - - def issue_entity(project) - if project.has_external_issue_tracker? - ::API::Entities::ExternalIssue - else - ::API::V3::Entities::Issue - end - end - - params :optional_params do - optional :description, type: String, desc: 'The description of the merge request' - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' - optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' - optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - end - end - - desc 'List merge requests' do - detail 'iid filter is deprecated have been removed on V4' - success ::API::V3::Entities::MergeRequest - end - params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return merge requests sorted in `asc` or `desc` order.' - optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' - use :pagination - end - get ":id/merge_requests" do - authorize! :read_merge_request, user_project - - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? - - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end - - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) - present paginate(merge_requests), with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project - end - - desc 'Create a merge request' do - success ::API::V3::Entities::MergeRequest - end - params do - requires :title, type: String, desc: 'The title of the merge request' - requires :source_branch, type: String, desc: 'The source branch' - requires :target_branch, type: String, desc: 'The target branch' - optional :target_project_id, type: Integer, - desc: 'The target project of the merge request defaults to the :id of the project' - use :optional_params - end - post ":id/merge_requests" do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126') - - authorize! :create_merge_request, user_project - - mr_params = declared_params(include_missing: false) - mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - - merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute - - if merge_request.valid? - present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project - else - handle_merge_request_errors! merge_request.errors - end - end - - desc 'Delete a merge request' - params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' - end - delete ":id/merge_requests/:merge_request_id" do - merge_request = find_project_merge_request(params[:merge_request_id]) - - authorize!(:destroy_merge_request, merge_request) - - status(200) - merge_request.destroy - end - - params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' - end - { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status| - desc 'Get a single merge request' do - if status == :deprecated - detail DEPRECATION_MESSAGE - end - - success ::API::V3::Entities::MergeRequest - end - get path do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - - present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project - end - - desc 'Get the commits of a merge request' do - success ::API::Entities::Commit - end - get "#{path}/commits" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - - present merge_request.commits, with: ::API::Entities::Commit - end - - desc 'Show the merge request changes' do - success ::API::Entities::MergeRequestChanges - end - get "#{path}/changes" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - - present merge_request, with: ::API::Entities::MergeRequestChanges, current_user: current_user - end - - desc 'Update a merge request' do - success ::API::V3::Entities::MergeRequest - end - params do - optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' - optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' - optional :state_event, type: String, values: %w[close reopen merge], - desc: 'Status of the merge request' - use :optional_params - at_least_one_of :title, :target_branch, :description, :assignee_id, - :milestone_id, :labels, :state_event, - :remove_source_branch - end - put path do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42127') - - merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) - - mr_params = declared_params(include_missing: false) - mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) - - if merge_request.valid? - present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project - else - handle_merge_request_errors! merge_request.errors - end - end - - desc 'Merge a merge request' do - success ::API::V3::Entities::MergeRequest - end - params do - optional :merge_commit_message, type: String, desc: 'Custom merge commit message' - optional :should_remove_source_branch, type: Boolean, - desc: 'When true, the source branch will be deleted if possible' - optional :merge_when_build_succeeds, type: Boolean, - desc: 'When true, this merge request will be merged when the build succeeds' - optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' - end - put "#{path}/merge" do - merge_request = find_project_merge_request(params[:merge_request_id]) - - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! unless merge_request.can_be_merged_by?(current_user) - - not_allowed! unless merge_request.mergeable_state? - - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? - - if params[:sha] && merge_request.diff_head_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) - end - - merge_params = { - commit_message: params[:merge_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] - } - - if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) - else - ::MergeRequests::MergeService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) - end - - present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project - end - - desc 'Cancel merge if "Merge When Build succeeds" is enabled' do - success ::API::V3::Entities::MergeRequest - end - post "#{path}/cancel_merge_when_build_succeeds" do - merge_request = find_project_merge_request(params[:merge_request_id]) - - unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) - - ::MergeRequest::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user) - .cancel(merge_request) - end - - desc 'Get the comments of a merge request' do - detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' - success ::API::Entities::MRNote - end - params do - use :pagination - end - get "#{path}/comments" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - present paginate(merge_request.notes.fresh), with: ::API::Entities::MRNote - end - - desc 'Post a comment to a merge request' do - detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' - success ::API::Entities::MRNote - end - params do - requires :note, type: String, desc: 'The text of the comment' - end - post "#{path}/comments" do - merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) - - opts = { - note: params[:note], - noteable_type: 'MergeRequest', - noteable_id: merge_request.id - } - - note = ::Notes::CreateService.new(user_project, current_user, opts).execute - - if note.save - present note, with: ::API::Entities::MRNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end - end - - desc 'List issues that will be closed on merge' do - success ::API::Entities::MRNote - end - params do - use :pagination - end - get "#{path}/closes_issues" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) - issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: issue_entity(user_project), current_user: current_user - end - end - end - end - end -end diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb deleted file mode 100644 index 9be4cf9d22a..00000000000 --- a/lib/api/v3/milestones.rb +++ /dev/null @@ -1,65 +0,0 @@ -module API - module V3 - class Milestones < Grape::API - include PaginationParams - - before { authenticate! } - - helpers do - def filter_milestones_state(milestones, state) - case state - when 'active' then milestones.active - when 'closed' then milestones.closed - else milestones - end - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a list of project milestones' do - success ::API::Entities::Milestone - end - params do - optional :state, type: String, values: %w[active closed all], default: 'all', - desc: 'Return "active", "closed", or "all" milestones' - optional :iid, type: Array[Integer], desc: 'The IID of the milestone' - use :pagination - end - get ":id/milestones" do - authorize! :read_milestone, user_project - - milestones = user_project.milestones - milestones = filter_milestones_state(milestones, params[:state]) - milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? - milestones = milestones.order_id_desc - - present paginate(milestones), with: ::API::Entities::Milestone - end - - desc 'Get all issues for a single project milestone' do - success ::API::V3::Entities::Issue - end - params do - requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' - use :pagination - end - get ':id/milestones/:milestone_id/issues' do - authorize! :read_milestone, user_project - - milestone = user_project.milestones.find(params[:milestone_id]) - - finder_params = { - project_id: user_project.id, - milestone_title: milestone.title - } - - issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project - end - end - end - end -end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb deleted file mode 100644 index d49772b92f2..00000000000 --- a/lib/api/v3/notes.rb +++ /dev/null @@ -1,148 +0,0 @@ -module API - module V3 - class Notes < Grape::API - include PaginationParams - - before { authenticate! } - - NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - NOTEABLE_TYPES.each do |noteable_type| - noteables_str = noteable_type.to_s.underscore.pluralize - - desc 'Get a list of project +noteable+ notes' do - success ::API::V3::Entities::Note - end - params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' - use :pagination - end - get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend - - if can?(current_user, noteable_read_ability_name(noteable), noteable) - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(noteable.notes) - .reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: ::API::V3::Entities::Note - else - not_found!("Notes") - end - end - - desc 'Get a single +noteable+ note' do - success ::API::V3::Entities::Note - end - params do - requires :note_id, type: Integer, desc: 'The ID of a note' - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' - end - get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend - note = noteable.notes.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) - - if can_read_note - present note, with: ::API::V3::Entities::Note - else - not_found!("Note") - end - end - - desc 'Create a new +noteable+ note' do - success ::API::V3::Entities::Note - end - params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' - requires :body, type: String, desc: 'The content of a note' - optional :created_at, type: String, desc: 'The creation date of the note' - end - post ":id/#{noteables_str}/:noteable_id/notes" do - opts = { - note: params[:body], - noteable_type: noteables_str.classify, - noteable_id: params[:noteable_id] - } - - noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend - - if can?(current_user, noteable_read_ability_name(noteable), noteable) - if params[:created_at] && (current_user.admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end - - note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if note.valid? - present note, with: ::API::V3::Entities.const_get(note.class.name) - else - not_found!("Note #{note.errors.messages}") - end - else - not_found!("Note") - end - end - - desc 'Update an existing +noteable+ note' do - success ::API::V3::Entities::Note - end - params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' - requires :note_id, type: Integer, desc: 'The ID of a note' - requires :body, type: String, desc: 'The content of a note' - end - put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - note = user_project.notes.find(params[:note_id]) - - authorize! :admin_note, note - - opts = { - note: params[:body] - } - - note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) - - if note.valid? - present note, with: ::API::V3::Entities::Note - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end - end - - desc 'Delete a +noteable+ note' do - success ::API::V3::Entities::Note - end - params do - requires :noteable_id, type: Integer, desc: 'The ID of the noteable' - requires :note_id, type: Integer, desc: 'The ID of a note' - end - delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - note = user_project.notes.find(params[:note_id]) - authorize! :admin_note, note - - ::Notes::DestroyService.new(user_project, current_user).execute(note) - - present note, with: ::API::V3::Entities::Note - end - end - end - - helpers do - def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore}".to_sym - end - end - end - end -end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb deleted file mode 100644 index 6d31c12f572..00000000000 --- a/lib/api/v3/pipelines.rb +++ /dev/null @@ -1,38 +0,0 @@ -module API - module V3 - class Pipelines < Grape::API - include PaginationParams - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The project ID' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get all Pipelines of the project' do - detail 'This feature was introduced in GitLab 8.11.' - success ::API::Entities::Pipeline - end - params do - use :pagination - optional :scope, type: String, values: %w(running branches tags), - desc: 'Either running, branches, or tags' - end - get ':id/pipelines' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42123') - - authorize! :read_pipeline, user_project - - pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute - present paginate(pipelines), with: ::API::Entities::Pipeline - end - end - - helpers do - def pipeline - @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) - end - end - end - end -end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb deleted file mode 100644 index 631944150c7..00000000000 --- a/lib/api/v3/project_hooks.rb +++ /dev/null @@ -1,111 +0,0 @@ -module API - module V3 - class ProjectHooks < Grape::API - include PaginationParams - - before { authenticate! } - before { authorize_admin_project } - - helpers do - params :project_hook_properties do - requires :url, type: String, desc: "The URL to send the request to" - optional :push_events, type: Boolean, desc: "Trigger hook on push events" - optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" - optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" - optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" - optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" - optional :build_events, type: Boolean, desc: "Trigger hook on build events" - optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" - optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" - optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" - optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get project hooks' do - success ::API::V3::Entities::ProjectHook - end - params do - use :pagination - end - get ":id/hooks" do - hooks = paginate user_project.hooks - - present hooks, with: ::API::V3::Entities::ProjectHook - end - - desc 'Get a project hook' do - success ::API::V3::Entities::ProjectHook - end - params do - requires :hook_id, type: Integer, desc: 'The ID of a project hook' - end - get ":id/hooks/:hook_id" do - hook = user_project.hooks.find(params[:hook_id]) - present hook, with: ::API::V3::Entities::ProjectHook - end - - desc 'Add hook to project' do - success ::API::V3::Entities::ProjectHook - end - params do - use :project_hook_properties - end - post ":id/hooks" do - attrs = declared_params(include_missing: false) - attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) - hook = user_project.hooks.new(attrs) - - if hook.save - present hook, with: ::API::V3::Entities::ProjectHook - else - error!("Invalid url given", 422) if hook.errors[:url].present? - - not_found!("Project hook #{hook.errors.messages}") - end - end - - desc 'Update an existing project hook' do - success ::API::V3::Entities::ProjectHook - end - params do - requires :hook_id, type: Integer, desc: "The ID of the hook to update" - use :project_hook_properties - end - put ":id/hooks/:hook_id" do - hook = user_project.hooks.find(params.delete(:hook_id)) - - attrs = declared_params(include_missing: false) - attrs[:job_events] = attrs.delete(:build_events) if attrs.key?(:build_events) - if hook.update_attributes(attrs) - present hook, with: ::API::V3::Entities::ProjectHook - else - error!("Invalid url given", 422) if hook.errors[:url].present? - - not_found!("Project hook #{hook.errors.messages}") - end - end - - desc 'Deletes project hook' do - success ::API::V3::Entities::ProjectHook - end - params do - requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' - end - delete ":id/hooks/:hook_id" do - begin - present user_project.hooks.destroy(params[:hook_id]), with: ::API::V3::Entities::ProjectHook - rescue - # ProjectHook can raise Error if hook_id not found - not_found!("Error deleting hook #{params[:hook_id]}") - end - end - end - end - end -end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb deleted file mode 100644 index 6ba425ba8c7..00000000000 --- a/lib/api/v3/project_snippets.rb +++ /dev/null @@ -1,143 +0,0 @@ -module API - module V3 - class ProjectSnippets < Grape::API - include PaginationParams - - before { authenticate! } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - helpers do - def handle_project_member_errors(errors) - if errors[:project_access].any? - error!(errors[:project_access], 422) - end - - not_found! - end - - def snippets_for_current_user - SnippetsFinder.new(current_user, project: user_project).execute - end - end - - desc 'Get all project snippets' do - success ::API::V3::Entities::ProjectSnippet - end - params do - use :pagination - end - get ":id/snippets" do - present paginate(snippets_for_current_user), with: ::API::V3::Entities::ProjectSnippet - end - - desc 'Get a single project snippet' do - success ::API::V3::Entities::ProjectSnippet - end - params do - requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' - end - get ":id/snippets/:snippet_id" do - snippet = snippets_for_current_user.find(params[:snippet_id]) - present snippet, with: ::API::V3::Entities::ProjectSnippet - end - - desc 'Create a new project snippet' do - success ::API::V3::Entities::ProjectSnippet - end - params do - requires :title, type: String, desc: 'The title of the snippet' - requires :file_name, type: String, desc: 'The file name of the snippet' - requires :code, type: String, desc: 'The content of the snippet' - requires :visibility_level, type: Integer, - values: [Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC], - desc: 'The visibility level of the snippet' - end - post ":id/snippets" do - authorize! :create_project_snippet, user_project - snippet_params = declared_params.merge(request: request, api: true) - snippet_params[:content] = snippet_params.delete(:code) - - snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute - - render_spam_error! if snippet.spam? - - if snippet.persisted? - present snippet, with: ::API::V3::Entities::ProjectSnippet - else - render_validation_error!(snippet) - end - end - - desc 'Update an existing project snippet' do - success ::API::V3::Entities::ProjectSnippet - end - params do - requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' - optional :title, type: String, desc: 'The title of the snippet' - optional :file_name, type: String, desc: 'The file name of the snippet' - optional :code, type: String, desc: 'The content of the snippet' - optional :visibility_level, type: Integer, - values: [Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC], - desc: 'The visibility level of the snippet' - at_least_one_of :title, :file_name, :code, :visibility_level - end - put ":id/snippets/:snippet_id" do - snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) - not_found!('Snippet') unless snippet - - authorize! :update_project_snippet, snippet - - snippet_params = declared_params(include_missing: false) - .merge(request: request, api: true) - - snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? - - UpdateSnippetService.new(user_project, current_user, snippet, - snippet_params).execute - - render_spam_error! if snippet.spam? - - if snippet.valid? - present snippet, with: ::API::V3::Entities::ProjectSnippet - else - render_validation_error!(snippet) - end - end - - desc 'Delete a project snippet' - params do - requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' - end - delete ":id/snippets/:snippet_id" do - snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) - not_found!('Snippet') unless snippet - - authorize! :admin_project_snippet, snippet - snippet.destroy - - status(200) - end - - desc 'Get a raw project snippet' - params do - requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' - end - get ":id/snippets/:snippet_id/raw" do - snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) - not_found!('Snippet') unless snippet - - env['api.format'] = :txt - content_type 'text/plain' - present snippet.content - end - end - end - end -end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb deleted file mode 100644 index 7d8b1f369fe..00000000000 --- a/lib/api/v3/projects.rb +++ /dev/null @@ -1,479 +0,0 @@ -module API - module V3 - class Projects < Grape::API - include PaginationParams - - before { authenticate_non_get! } - - after_validation do - set_only_allow_merge_if_pipeline_succeeds! - end - - helpers do - params :optional_params do - optional :description, type: String, desc: 'The description of the project' - optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' - optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' - optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' - optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' - optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' - optional :visibility_level, type: Integer, values: [ - Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC - ], desc: 'Create a public project. The same as visibility_level = 20.' - optional :public_builds, type: Boolean, desc: 'Perform public builds' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - end - - def map_public_to_visibility_level(attrs) - publik = attrs.delete(:public) - if !publik.nil? && !attrs[:visibility_level].present? - # Since setting the public attribute to private could mean either - # private or internal, use the more conservative option, private. - attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE - end - - attrs - end - - def set_only_allow_merge_if_pipeline_succeeds! - if params.key?(:only_allow_merge_if_build_succeeds) - params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) - end - end - end - - resource :projects do - helpers do - params :collection_params do - use :sort_params - use :filter_params - use :pagination - - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - end - - params :sort_params do - optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], - default: 'created_at', desc: 'Return projects ordered by field' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return projects sorted in ascending and descending order' - end - - params :filter_params do - optional :archived, type: Boolean, default: nil, desc: 'Limit by archived status' - optional :visibility, type: String, values: %w[public internal private], - desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' - end - - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - - params :create_params do - optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' - optional :import_url, type: String, desc: 'URL from which the project is imported' - end - - def present_projects(projects, options = {}) - options = options.reverse_merge( - with: ::API::V3::Entities::Project, - current_user: current_user, - simple: params[:simple] - ) - - projects = filter_projects(projects) - projects = projects.with_statistics if options[:statistics] - options[:with] = ::API::Entities::BasicProjectDetails if options[:simple] - - present paginate(projects), options - end - end - - desc 'Get a list of visible projects for authenticated user' do - success ::API::Entities::BasicProjectDetails - end - params do - use :collection_params - end - get '/visible' do - entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity - end - - desc 'Get a projects list for authenticated user' do - success ::API::Entities::BasicProjectDetails - end - params do - use :collection_params - end - get do - authenticate! - - present_projects current_user.authorized_projects.order_id_desc, - with: ::API::V3::Entities::ProjectWithAccess - end - - desc 'Get an owned projects list for authenticated user' do - success ::API::Entities::BasicProjectDetails - end - params do - use :collection_params - use :statistics_params - end - get '/owned' do - authenticate! - - present_projects current_user.owned_projects, - with: ::API::V3::Entities::ProjectWithAccess, - statistics: params[:statistics] - end - - desc 'Gets starred project for the authenticated user' do - success ::API::Entities::BasicProjectDetails - end - params do - use :collection_params - end - get '/starred' do - authenticate! - - present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute - end - - desc 'Get all projects for admin user' do - success ::API::Entities::BasicProjectDetails - end - params do - use :collection_params - use :statistics_params - end - get '/all' do - authenticated_as_admin! - - present_projects Project.all, with: ::API::V3::Entities::ProjectWithAccess, statistics: params[:statistics] - end - - desc 'Search for projects the current user has access to' do - success ::API::V3::Entities::Project - end - params do - requires :query, type: String, desc: 'The project name to be searched' - use :sort_params - use :pagination - end - get "/search/:query", requirements: { query: %r{[^/]+} } do - search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute - projects = search_service.objects('projects', params[:page], false) - projects = projects.reorder(params[:order_by] => params[:sort]) - - present paginate(projects), with: ::API::V3::Entities::Project - end - - desc 'Create new project' do - success ::API::V3::Entities::Project - end - params do - optional :name, type: String, desc: 'The name of the project' - optional :path, type: String, desc: 'The path of the repository' - at_least_one_of :name, :path - use :optional_params - use :create_params - end - post do - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) - project = ::Projects::CreateService.new(current_user, attrs).execute - - if project.saved? - present project, with: ::API::V3::Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project) - else - if project.errors[:limit_reached].present? - error!(project.errors[:limit_reached], 403) - end - - render_validation_error!(project) - end - end - - desc 'Create new project for a specified user. Only available to admin users.' do - success ::API::V3::Entities::Project - end - params do - requires :name, type: String, desc: 'The name of the project' - requires :user_id, type: Integer, desc: 'The ID of a user' - optional :default_branch, type: String, desc: 'The default branch of the project' - use :optional_params - use :create_params - end - post "user/:user_id" do - authenticated_as_admin! - user = User.find_by(id: params.delete(:user_id)) - not_found!('User') unless user - - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) - project = ::Projects::CreateService.new(user, attrs).execute - - if project.saved? - present project, with: ::API::V3::Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project) - else - render_validation_error!(project) - end - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a single project' do - success ::API::V3::Entities::ProjectWithAccess - end - get ":id" do - entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) - end - - desc 'Get events for a single project' do - success ::API::V3::Entities::Event - end - params do - use :pagination - end - get ":id/events" do - present paginate(user_project.events.recent), with: ::API::V3::Entities::Event - end - - desc 'Fork new project for the current user or provided namespace.' do - success ::API::V3::Entities::Project - end - params do - optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' - end - post 'fork/:id' do - fork_params = declared_params(include_missing: false) - namespace_id = fork_params[:namespace] - - if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end - - unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) - not_found!('Target Namespace') - end - end - - forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute - - if forked_project.errors.any? - conflict!(forked_project.errors.messages) - else - present forked_project, with: ::API::V3::Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, forked_project) - end - end - - desc 'Update an existing project' do - success ::API::V3::Entities::Project - end - params do - optional :name, type: String, desc: 'The name of the project' - optional :default_branch, type: String, desc: 'The default branch of the project' - optional :path, type: String, desc: 'The path of the repository' - use :optional_params - at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, - :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :resolve_outdated_diff_discussions, - :container_registry_enabled, :lfs_enabled, :public, :visibility_level, - :public_builds, :request_access_enabled, :only_allow_merge_if_build_succeeds, - :only_allow_merge_if_all_discussions_are_resolved, :path, - :default_branch - end - put ':id' do - authorize_admin_project - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) - authorize! :rename_project, user_project if attrs[:name].present? - authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? - - result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute - - if result[:status] == :success - present user_project, with: ::API::V3::Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, user_project) - else - render_validation_error!(user_project) - end - end - - desc 'Archive a project' do - success ::API::V3::Entities::Project - end - post ':id/archive' do - authorize!(:archive_project, user_project) - - user_project.archive! - - present user_project, with: ::API::V3::Entities::Project - end - - desc 'Unarchive a project' do - success ::API::V3::Entities::Project - end - post ':id/unarchive' do - authorize!(:archive_project, user_project) - - user_project.unarchive! - - present user_project, with: ::API::V3::Entities::Project - end - - desc 'Star a project' do - success ::API::V3::Entities::Project - end - post ':id/star' do - if current_user.starred?(user_project) - not_modified! - else - current_user.toggle_star(user_project) - user_project.reload - - present user_project, with: ::API::V3::Entities::Project - end - end - - desc 'Unstar a project' do - success ::API::V3::Entities::Project - end - delete ':id/star' do - if current_user.starred?(user_project) - current_user.toggle_star(user_project) - user_project.reload - - present user_project, with: ::API::V3::Entities::Project - else - not_modified! - end - end - - desc 'Remove a project' - delete ":id" do - authorize! :remove_project, user_project - - status(200) - ::Projects::DestroyService.new(user_project, current_user, {}).async_execute - end - - desc 'Mark this project as forked from another' - params do - requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' - end - post ":id/fork/:forked_from_id" do - authenticated_as_admin! - - forked_from_project = find_project!(params[:forked_from_id]) - not_found!("Source Project") unless forked_from_project - - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) - - ::Projects::ForksCountService.new(forked_from_project).refresh_cache - else - render_api_error!("Project already forked", 409) - end - end - - desc 'Remove a forked_from relationship' - delete ":id/fork" do - authorize! :remove_fork_project, user_project - - if user_project.forked? - status(200) - user_project.forked_project_link.destroy - else - not_modified! - end - end - - desc 'Share the project with a group' do - success ::API::Entities::ProjectGroupLink - end - params do - requires :group_id, type: Integer, desc: 'The ID of a group' - requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' - optional :expires_at, type: Date, desc: 'Share expiration date' - end - post ":id/share" do - authorize! :admin_project, user_project - group = Group.find_by_id(params[:group_id]) - - unless group && can?(current_user, :read_group, group) - not_found!('Group') - end - - unless user_project.allowed_to_share_with_group? - return render_api_error!("The project sharing with group is disabled", 400) - end - - link = user_project.project_group_links.new(declared_params(include_missing: false)) - - if link.save - present link, with: ::API::Entities::ProjectGroupLink - else - render_api_error!(link.errors.full_messages.first, 409) - end - end - - params do - requires :group_id, type: Integer, desc: 'The ID of the group' - end - delete ":id/share/:group_id" do - authorize! :admin_project, user_project - - link = user_project.project_group_links.find_by(group_id: params[:group_id]) - not_found!('Group Link') unless link - - link.destroy - no_content! - end - - desc 'Upload a file' - params do - requires :file, type: File, desc: 'The file to be uploaded' - end - post ":id/uploads" do - UploadService.new(user_project, params[:file]).execute - end - - desc 'Get the users list of a project' do - success ::API::Entities::UserBasic - end - params do - optional :search, type: String, desc: 'Return list of users matching the search criteria' - use :pagination - end - get ':id/users' do - users = user_project.team.users - users = users.search(params[:search]) if params[:search].present? - - present paginate(users), with: ::API::Entities::UserBasic - end - end - end - end -end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb deleted file mode 100644 index 5b54734bb45..00000000000 --- a/lib/api/v3/repositories.rb +++ /dev/null @@ -1,110 +0,0 @@ -require 'mime/types' - -module API - module V3 - class Repositories < Grape::API - before { authorize! :download_code, user_project } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - helpers do - def handle_project_member_errors(errors) - if errors[:project_access].any? - error!(errors[:project_access], 422) - end - - not_found! - end - end - - desc 'Get a project repository tree' do - success ::API::Entities::TreeObject - end - params do - optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :path, type: String, desc: 'The path of the tree' - optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' - end - get ':id/repository/tree' do - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - path = params[:path] || nil - - commit = user_project.commit(ref) - not_found!('Tree') unless commit - - tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - - present tree.sorted_entries, with: ::API::Entities::TreeObject - end - - desc 'Get a raw file contents' - params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' - requires :filepath, type: String, desc: 'The path to the file to display' - end - get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"], requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - repo = user_project.repository - commit = repo.commit(params[:sha]) - not_found! "Commit" unless commit - blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) - not_found! "File" unless blob - send_git_blob repo, blob - end - - desc 'Get a raw blob contents by blob sha' - params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' - end - get ':id/repository/raw_blobs/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - repo = user_project.repository - begin - blob = Gitlab::Git::Blob.raw(repo, params[:sha]) - rescue - not_found! 'Blob' - end - not_found! 'Blob' unless blob - send_git_blob repo, blob - end - - desc 'Get an archive of the repository' - params do - optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' - optional :format, type: String, desc: 'The archive format' - end - get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do - begin - send_git_archive user_project.repository, ref: params[:sha], format: params[:format] - rescue - not_found!('File') - end - end - - desc 'Compare two branches, tags, or commits' do - success ::API::Entities::Compare - end - params do - requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' - requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' - end - get ':id/repository/compare' do - compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) - present compare, with: ::API::Entities::Compare - end - - desc 'Get repository contributors' do - success ::API::Entities::Contributor - end - get ':id/repository/contributors' do - begin - present user_project.repository.contributors, - with: ::API::Entities::Contributor - rescue - not_found! - end - end - end - end - end -end diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb deleted file mode 100644 index c6d9957d452..00000000000 --- a/lib/api/v3/runners.rb +++ /dev/null @@ -1,66 +0,0 @@ -module API - module V3 - class Runners < Grape::API - include PaginationParams - - before { authenticate! } - - resource :runners do - desc 'Remove a runner' do - success ::API::Entities::Runner - end - params do - requires :id, type: Integer, desc: 'The ID of the runner' - end - delete ':id' do - runner = Ci::Runner.find(params[:id]) - not_found!('Runner') unless runner - - authenticate_delete_runner!(runner) - - status(200) - runner.destroy - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - before { authorize_admin_project } - - desc "Disable project's runner" do - success ::API::Entities::Runner - end - params do - requires :runner_id, type: Integer, desc: 'The ID of the runner' - end - delete ':id/runners/:runner_id' do - runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) - not_found!('Runner') unless runner_project - - runner = runner_project.runner - forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 - - runner_project.destroy - - present runner, with: ::API::Entities::Runner - end - end - - helpers do - def authenticate_delete_runner!(runner) - return if current_user.admin? - - forbidden!("Runner is shared") if runner.is_shared? - forbidden!("Runner associated with more than one project") if runner.projects.count > 1 - forbidden!("No access granted") unless user_can_access_runner?(runner) - end - - def user_can_access_runner?(runner) - current_user.ci_authorized_runners.exists?(runner.id) - end - end - end - end -end diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb deleted file mode 100644 index 20ca1021c71..00000000000 --- a/lib/api/v3/services.rb +++ /dev/null @@ -1,670 +0,0 @@ -module API - module V3 - class Services < Grape::API - services = { - 'asana' => [ - { - required: true, - name: :api_key, - type: String, - desc: 'User API token' - }, - { - required: false, - name: :restrict_to_branch, - type: String, - desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' - } - ], - 'assembla' => [ - { - required: true, - name: :token, - type: String, - desc: 'The authentication token' - }, - { - required: false, - name: :subdomain, - type: String, - desc: 'Subdomain setting' - } - ], - 'bamboo' => [ - { - required: true, - name: :bamboo_url, - type: String, - desc: 'Bamboo root URL like https://bamboo.example.com' - }, - { - required: true, - name: :build_key, - type: String, - desc: 'Bamboo build plan key like' - }, - { - required: true, - name: :username, - type: String, - desc: 'A user with API access, if applicable' - }, - { - required: true, - name: :password, - type: String, - desc: 'Passord of the user' - } - ], - 'bugzilla' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'New issue URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'Issues URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'Project URL' - }, - { - required: false, - name: :description, - type: String, - desc: 'Description' - }, - { - required: false, - name: :title, - type: String, - desc: 'Title' - } - ], - 'buildkite' => [ - { - required: true, - name: :token, - type: String, - desc: 'Buildkite project GitLab token' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'The buildkite project URL' - }, - { - required: false, - name: :enable_ssl_verification, - type: Boolean, - desc: 'Enable SSL verification for communication' - } - ], - 'builds-email' => [ - { - required: true, - name: :recipients, - type: String, - desc: 'Comma-separated list of recipient email addresses' - }, - { - required: false, - name: :add_pusher, - type: Boolean, - desc: 'Add pusher to recipients list' - }, - { - required: false, - name: :notify_only_broken_builds, - type: Boolean, - desc: 'Notify only broken builds' - } - ], - 'campfire' => [ - { - required: true, - name: :token, - type: String, - desc: 'Campfire token' - }, - { - required: false, - name: :subdomain, - type: String, - desc: 'Campfire subdomain' - }, - { - required: false, - name: :room, - type: String, - desc: 'Campfire room' - } - ], - 'custom-issue-tracker' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'New issue URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'Issues URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'Project URL' - }, - { - required: false, - name: :description, - type: String, - desc: 'Description' - }, - { - required: false, - name: :title, - type: String, - desc: 'Title' - } - ], - 'drone-ci' => [ - { - required: true, - name: :token, - type: String, - desc: 'Drone CI token' - }, - { - required: true, - name: :drone_url, - type: String, - desc: 'Drone CI URL' - }, - { - required: false, - name: :enable_ssl_verification, - type: Boolean, - desc: 'Enable SSL verification for communication' - } - ], - 'emails-on-push' => [ - { - required: true, - name: :recipients, - type: String, - desc: 'Comma-separated list of recipient email addresses' - }, - { - required: false, - name: :disable_diffs, - type: Boolean, - desc: 'Disable code diffs' - }, - { - required: false, - name: :send_from_committer_email, - type: Boolean, - desc: 'Send from committer' - } - ], - 'external-wiki' => [ - { - required: true, - name: :external_wiki_url, - type: String, - desc: 'The URL of the external Wiki' - } - ], - 'flowdock' => [ - { - required: true, - name: :token, - type: String, - desc: 'Flowdock token' - } - ], - 'gemnasium' => [ - { - required: true, - name: :api_key, - type: String, - desc: 'Your personal API key on gemnasium.com' - }, - { - required: true, - name: :token, - type: String, - desc: "The project's slug on gemnasium.com" - } - ], - 'hipchat' => [ - { - required: true, - name: :token, - type: String, - desc: 'The room token' - }, - { - required: false, - name: :room, - type: String, - desc: 'The room name or ID' - }, - { - required: false, - name: :color, - type: String, - desc: 'The room color' - }, - { - required: false, - name: :notify, - type: Boolean, - desc: 'Enable notifications' - }, - { - required: false, - name: :api_version, - type: String, - desc: 'Leave blank for default (v2)' - }, - { - required: false, - name: :server, - type: String, - desc: 'Leave blank for default. https://hipchat.example.com' - } - ], - 'irker' => [ - { - required: true, - name: :recipients, - type: String, - desc: 'Recipients/channels separated by whitespaces' - }, - { - required: false, - name: :default_irc_uri, - type: String, - desc: 'Default: irc://irc.network.net:6697' - }, - { - required: false, - name: :server_host, - type: String, - desc: 'Server host. Default localhost' - }, - { - required: false, - name: :server_port, - type: Integer, - desc: 'Server port. Default 6659' - }, - { - required: false, - name: :colorize_messages, - type: Boolean, - desc: 'Colorize messages' - } - ], - 'jira' => [ - { - required: true, - name: :url, - type: String, - desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' - }, - { - required: true, - name: :project_key, - type: String, - desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' - }, - { - required: false, - name: :username, - type: String, - desc: 'The username of the user created to be used with GitLab/JIRA' - }, - { - required: false, - name: :password, - type: String, - desc: 'The password of the user created to be used with GitLab/JIRA' - }, - { - required: false, - name: :jira_issue_transition_id, - type: Integer, - desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' - } - ], - - 'kubernetes' => [ - { - required: true, - name: :namespace, - type: String, - desc: 'The Kubernetes namespace to use' - }, - { - required: true, - name: :api_url, - type: String, - desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' - }, - { - required: true, - name: :token, - type: String, - desc: 'The service token to authenticate against the Kubernetes cluster with' - }, - { - required: false, - name: :ca_pem, - type: String, - desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - } - ], - 'mattermost-slash-commands' => [ - { - required: true, - name: :token, - type: String, - desc: 'The Mattermost token' - } - ], - 'slack-slash-commands' => [ - { - required: true, - name: :token, - type: String, - desc: 'The Slack token' - } - ], - 'packagist' => [ - { - required: true, - name: :username, - type: String, - desc: 'The username' - }, - { - required: true, - name: :token, - type: String, - desc: 'The Packagist API token' - }, - { - required: false, - name: :server, - type: String, - desc: 'The server' - } - ], - 'pipelines-email' => [ - { - required: true, - name: :recipients, - type: String, - desc: 'Comma-separated list of recipient email addresses' - }, - { - required: false, - name: :notify_only_broken_builds, - type: Boolean, - desc: 'Notify only broken builds' - } - ], - 'pivotaltracker' => [ - { - required: true, - name: :token, - type: String, - desc: 'The Pivotaltracker token' - }, - { - required: false, - name: :restrict_to_branch, - type: String, - desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' - } - ], - 'pushover' => [ - { - required: true, - name: :api_key, - type: String, - desc: 'The application key' - }, - { - required: true, - name: :user_key, - type: String, - desc: 'The user key' - }, - { - required: true, - name: :priority, - type: String, - desc: 'The priority' - }, - { - required: true, - name: :device, - type: String, - desc: 'Leave blank for all active devices' - }, - { - required: true, - name: :sound, - type: String, - desc: 'The sound of the notification' - } - ], - 'redmine' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'The new issue URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'The project URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'The issues URL' - }, - { - required: false, - name: :description, - type: String, - desc: 'The description of the tracker' - } - ], - 'slack' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' - }, - { - required: false, - name: :new_issue_url, - type: String, - desc: 'The user name' - }, - { - required: false, - name: :channel, - type: String, - desc: 'The channel name' - } - ], - 'microsoft-teams' => [ - required: true, - name: :webhook, - type: String, - desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' - ], - 'mattermost' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' - } - ], - 'teamcity' => [ - { - required: true, - name: :teamcity_url, - type: String, - desc: 'TeamCity root URL like https://teamcity.example.com' - }, - { - required: true, - name: :build_type, - type: String, - desc: 'Build configuration ID' - }, - { - required: true, - name: :username, - type: String, - desc: 'A user with permissions to trigger a manual build' - }, - { - required: true, - name: :password, - type: String, - desc: 'The password of the user' - } - ] - } - - trigger_services = { - 'mattermost-slash-commands' => [ - { - name: :token, - type: String, - desc: 'The Mattermost token' - } - ], - 'slack-slash-commands' => [ - { - name: :token, - type: String, - desc: 'The Slack token' - } - ] - }.freeze - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - before { authenticate! } - before { authorize_admin_project } - - helpers do - def service_attributes(service) - service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym - end - end - end - - desc "Delete a service for project" - params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' - end - delete ":id/services/:service_slug" do - service = user_project.find_or_initialize_service(params[:service_slug].underscore) - - attrs = service_attributes(service).inject({}) do |hash, key| - hash.merge!(key => nil) - end - - if service.update_attributes(attrs.merge(active: false)) - status(200) - true - else - render_api_error!('400 Bad Request', 400) - end - end - - desc 'Get the service settings for project' do - success Entities::ProjectService - end - params do - requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' - end - get ":id/services/:service_slug" do - service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService - end - end - - trigger_services.each do |service_slug, settings| - helpers do - def slash_command_service(project, service_slug, params) - project.services.active.where(template: false).find do |service| - service.try(:token) == params[:token] && service.to_param == service_slug.underscore - end - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc "Trigger a slash command for #{service_slug}" do - detail 'Added in GitLab 8.13' - end - params do - settings.each do |setting| - requires setting[:name], type: setting[:type], desc: setting[:desc] - end - end - post ":id/services/#{service_slug.underscore}/trigger" do - project = find_project(params[:id]) - - # This is not accurate, but done to prevent leakage of the project names - not_found!('Service') unless project - - service = slash_command_service(project, service_slug, params) - result = service.try(:trigger, params) - - if result - status result[:status] || 200 - present result - else - not_found!('Service') - end - end - end - end - end - end -end diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb deleted file mode 100644 index 9b4ab7630fb..00000000000 --- a/lib/api/v3/settings.rb +++ /dev/null @@ -1,147 +0,0 @@ -module API - module V3 - class Settings < Grape::API - before { authenticated_as_admin! } - - helpers do - def current_settings - @current_setting ||= - (ApplicationSetting.current || ApplicationSetting.create_from_defaults) - end - end - - desc 'Get the current application settings' do - success Entities::ApplicationSetting - end - get "application/settings" do - present current_settings, with: Entities::ApplicationSetting - end - - desc 'Modify application settings' do - success Entities::ApplicationSetting - end - params do - optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' - optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' - optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' - optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' - optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], - desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' - optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' - optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' - optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' - optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' - optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' - optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' - optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' - optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' - optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' - optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' - optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' - optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' - given domain_blacklist_enabled: ->(val) { val } do - requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' - end - optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' - mutually_exclusive :password_authentication_enabled, :signin_enabled - optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' - given require_two_factor_authentication: ->(val) { val } do - requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' - end - optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' - optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' - optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' - optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' - optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' - given shared_runners_enabled: ->(val) { val } do - requires :shared_runners_text, type: String, desc: 'Shared runners text ' - end - optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have" - optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' - optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' - optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' - given metrics_enabled: ->(val) { val } do - requires :metrics_host, type: String, desc: 'The InfluxDB host' - requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' - requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' - requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' - requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' - requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' - requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' - end - optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' - given sidekiq_throttling_enabled: ->(val) { val } do - requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' - requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' - end - optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' - given recaptcha_enabled: ->(val) { val } do - requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' - requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' - end - optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' - given akismet_enabled: ->(val) { val } do - requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' - end - optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' - optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' - given sentry_enabled: ->(val) { val } do - requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' - end - optional :repository_storage, type: String, desc: 'Storage paths for new projects' - optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." - optional :koding_enabled, type: Boolean, desc: 'Enable Koding' - given koding_enabled: ->(val) { val } do - requires :koding_url, type: String, desc: 'The Koding team URL' - end - optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' - given plantuml_enabled: ->(val) { val } do - requires :plantuml_url, type: String, desc: 'The PlantUML server URL' - end - optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' - optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' - optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' - optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' - given housekeeping_enabled: ->(val) { val } do - requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." - requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." - requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." - requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." - end - optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' - at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, - :default_group_visibility, :restricted_visibility_levels, :import_sources, - :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, - :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, - :user_oauth_applications, :user_default_external, :signup_enabled, - :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication, - :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, - :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, - :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, - :akismet_enabled, :admin_notification_email, :sentry_enabled, - :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, - :version_check_enabled, :email_author_in_body, :html_emails_enabled, - :housekeeping_enabled, :terminal_max_session_time - end - put "application/settings" do - attrs = declared_params(include_missing: false) - - if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) - elsif attrs.has_key?(:password_authentication_enabled) - attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) - end - - if current_settings.update_attributes(attrs) - present current_settings, with: Entities::ApplicationSetting - else - render_validation_error!(current_settings) - end - end - end - end -end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb deleted file mode 100644 index 85613c8ed84..00000000000 --- a/lib/api/v3/snippets.rb +++ /dev/null @@ -1,141 +0,0 @@ -module API - module V3 - class Snippets < Grape::API - include PaginationParams - - before { authenticate! } - - resource :snippets do - helpers do - def snippets_for_current_user - SnippetsFinder.new(current_user, author: current_user).execute - end - - def public_snippets - SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute - end - end - - desc 'Get a snippets list for authenticated user' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - use :pagination - end - get do - present paginate(snippets_for_current_user), with: ::API::Entities::PersonalSnippet - end - - desc 'List all public snippets current_user has access to' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - use :pagination - end - get 'public' do - present paginate(public_snippets), with: ::API::Entities::PersonalSnippet - end - - desc 'Get a single snippet' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - requires :id, type: Integer, desc: 'The ID of a snippet' - end - get ':id' do - snippet = snippets_for_current_user.find(params[:id]) - present snippet, with: ::API::Entities::PersonalSnippet - end - - desc 'Create new snippet' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - requires :title, type: String, desc: 'The title of a snippet' - requires :file_name, type: String, desc: 'The name of a snippet file' - requires :content, type: String, desc: 'The content of a snippet' - optional :visibility_level, type: Integer, - values: Gitlab::VisibilityLevel.values, - default: Gitlab::VisibilityLevel::INTERNAL, - desc: 'The visibility level of the snippet' - end - post do - attrs = declared_params(include_missing: false).merge(request: request, api: true) - snippet = CreateSnippetService.new(nil, current_user, attrs).execute - - if snippet.persisted? - present snippet, with: ::API::Entities::PersonalSnippet - else - render_validation_error!(snippet) - end - end - - desc 'Update an existing snippet' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - requires :id, type: Integer, desc: 'The ID of a snippet' - optional :title, type: String, desc: 'The title of a snippet' - optional :file_name, type: String, desc: 'The name of a snippet file' - optional :content, type: String, desc: 'The content of a snippet' - optional :visibility_level, type: Integer, - values: Gitlab::VisibilityLevel.values, - desc: 'The visibility level of the snippet' - at_least_one_of :title, :file_name, :content, :visibility_level - end - put ':id' do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet - - authorize! :update_personal_snippet, snippet - - attrs = declared_params(include_missing: false) - - UpdateSnippetService.new(nil, current_user, snippet, attrs).execute - - if snippet.persisted? - present snippet, with: ::API::Entities::PersonalSnippet - else - render_validation_error!(snippet) - end - end - - desc 'Remove snippet' do - detail 'This feature was introduced in GitLab 8.15.' - success ::API::Entities::PersonalSnippet - end - params do - requires :id, type: Integer, desc: 'The ID of a snippet' - end - delete ':id' do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet - - authorize! :destroy_personal_snippet, snippet - snippet.destroy - no_content! - end - - desc 'Get a raw snippet' do - detail 'This feature was introduced in GitLab 8.15.' - end - params do - requires :id, type: Integer, desc: 'The ID of a snippet' - end - get ":id/raw" do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet - - env['api.format'] = :txt - content_type 'text/plain' - present snippet.content - end - end - end - end -end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb deleted file mode 100644 index 690768db82f..00000000000 --- a/lib/api/v3/subscriptions.rb +++ /dev/null @@ -1,53 +0,0 @@ -module API - module V3 - class Subscriptions < Grape::API - before { authenticate! } - - subscribable_types = { - 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, - 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, - 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) } - } - - params do - requires :id, type: String, desc: 'The ID of a project' - requires :subscribable_id, type: String, desc: 'The ID of a resource' - end - resource :projects, requirements: { id: %r{[^/]+} } do - subscribable_types.each do |type, finder| - type_singularized = type.singularize - entity_class = ::API::Entities.const_get(type_singularized.camelcase) - - desc 'Subscribe to a resource' do - success entity_class - end - post ":id/#{type}/:subscribable_id/subscription" do - resource = instance_exec(params[:subscribable_id], &finder) - - if resource.subscribed?(current_user, user_project) - not_modified! - else - resource.subscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project - end - end - - desc 'Unsubscribe from a resource' do - success entity_class - end - delete ":id/#{type}/:subscribable_id/subscription" do - resource = instance_exec(params[:subscribable_id], &finder) - - if !resource.subscribed?(current_user, user_project) - not_modified! - else - resource.unsubscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project - end - end - end - end - end - end -end diff --git a/lib/api/v3/system_hooks.rb b/lib/api/v3/system_hooks.rb deleted file mode 100644 index 5787c06fc12..00000000000 --- a/lib/api/v3/system_hooks.rb +++ /dev/null @@ -1,32 +0,0 @@ -module API - module V3 - class SystemHooks < Grape::API - before do - authenticate! - authenticated_as_admin! - end - - resource :hooks do - desc 'Get the list of system hooks' do - success ::API::Entities::Hook - end - get do - present SystemHook.all, with: ::API::Entities::Hook - end - - desc 'Delete a hook' do - success ::API::Entities::Hook - end - params do - requires :id, type: Integer, desc: 'The ID of the system hook' - end - delete ":id" do - hook = SystemHook.find_by(id: params[:id]) - not_found!('System hook') unless hook - - present hook.destroy, with: ::API::Entities::Hook - end - end - end - end -end diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb deleted file mode 100644 index 6e37d31d153..00000000000 --- a/lib/api/v3/tags.rb +++ /dev/null @@ -1,40 +0,0 @@ -module API - module V3 - class Tags < Grape::API - before { authorize! :download_code, user_project } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a project repository tags' do - success ::API::Entities::Tag - end - get ":id/repository/tags" do - tags = user_project.repository.tags.sort_by(&:name).reverse - present tags, with: ::API::Entities::Tag, project: user_project - end - - desc 'Delete a repository tag' - params do - requires :tag_name, type: String, desc: 'The name of the tag' - end - delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do - authorize_push_project - - result = ::Tags::DestroyService.new(user_project, current_user) - .execute(params[:tag_name]) - - if result[:status] == :success - status(200) - { - tag_name: params[:tag_name] - } - else - render_api_error!(result[:message], result[:return_code]) - end - end - end - end - end -end diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb deleted file mode 100644 index b82b02b5f49..00000000000 --- a/lib/api/v3/templates.rb +++ /dev/null @@ -1,122 +0,0 @@ -module API - module V3 - class Templates < Grape::API - GLOBAL_TEMPLATE_TYPES = { - gitignores: { - klass: Gitlab::Template::GitignoreTemplate, - gitlab_version: 8.8 - }, - gitlab_ci_ymls: { - klass: Gitlab::Template::GitlabCiYmlTemplate, - gitlab_version: 8.9 - }, - dockerfiles: { - klass: Gitlab::Template::DockerfileTemplate, - gitlab_version: 8.15 - } - }.freeze - PROJECT_TEMPLATE_REGEX = - %r{[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]}xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - %r{[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]}xi.freeze - DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze - - helpers do - def parsed_license_template - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - template = Licensee::License.new(params[:name]) - - template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - template - end - - def render_response(template_type, template) - not_found!(template_type.to_s.singularize) unless template - present template, with: ::API::Entities::Template - end - end - - { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| - desc 'Get the list of the available license template' do - detailed_desc = 'This feature was introduced in GitLab 8.7.' - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success ::API::Entities::License - end - params do - optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' - end - get route do - options = { - featured: declared(params)[:popular].present? ? true : nil - } - present Licensee::License.all(options), with: ::API::Entities::License - end - end - - { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| - desc 'Get the text for a specific license' do - detailed_desc = 'This feature was introduced in GitLab 8.7.' - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success ::API::Entities::License - end - params do - requires :name, type: String, desc: 'The name of the template' - end - get route, requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params)[:name]) - - template = parsed_license_template - - present template, with: ::API::Entities::License - end - end - - GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| - klass = properties[:klass] - gitlab_version = properties[:gitlab_version] - - { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| - desc 'Get the list of the available template' do - detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success ::API::Entities::TemplatesList - end - get route do - present klass.all, with: ::API::Entities::TemplatesList - end - end - - { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| - desc 'Get the text for a specific template present in local filesystem' do - detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success ::API::Entities::Template - end - params do - requires :name, type: String, desc: 'The name of the template' - end - get route do - new_template = klass.find(declared(params)[:name]) - - render_response(template_type, new_template) - end - end - end - end - end -end diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb deleted file mode 100644 index 1aad39815f9..00000000000 --- a/lib/api/v3/time_tracking_endpoints.rb +++ /dev/null @@ -1,116 +0,0 @@ -module API - module V3 - module TimeTrackingEndpoints - extend ActiveSupport::Concern - - included do - helpers do - def issuable_name - declared_params.key?(:issue_id) ? 'issue' : 'merge_request' - end - - def issuable_key - "#{issuable_name}_id".to_sym - end - - def update_issuable_key - "update_#{issuable_name}".to_sym - end - - def read_issuable_key - "read_#{issuable_name}".to_sym - end - - def load_issuable - @issuable ||= begin - case issuable_name - when 'issue' - find_project_issue(params.delete(issuable_key)) - when 'merge_request' - find_project_merge_request(params.delete(issuable_key)) - end - end - end - - def update_issuable(attrs) - custom_params = declared_params(include_missing: false) - custom_params.merge!(attrs) - - issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) - if issuable.valid? - present issuable, with: ::API::Entities::IssuableTimeStats - else - render_validation_error!(issuable) - end - end - - def update_service - issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService - end - end - - issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' - issuable_collection_name = issuable_name.pluralize - issuable_key = "#{issuable_name}_id".to_sym - - desc "Set a time estimate for a project #{issuable_name}" - params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - requires :duration, type: String, desc: 'The duration to be parsed' - end - post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do - authorize! update_issuable_key, load_issuable - - status :ok - update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) - end - - desc "Reset the time estimate for a project #{issuable_name}" - params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - end - post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do - authorize! update_issuable_key, load_issuable - - status :ok - update_issuable(time_estimate: 0) - end - - desc "Add spent time for a project #{issuable_name}" - params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - requires :duration, type: String, desc: 'The duration to be parsed' - end - post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do - authorize! update_issuable_key, load_issuable - - update_issuable(spend_time: { - duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), - user_id: current_user.id - }) - end - - desc "Reset spent time for a project #{issuable_name}" - params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - end - post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do - authorize! update_issuable_key, load_issuable - - status :ok - update_issuable(spend_time: { duration: :reset, user_id: current_user.id }) - end - - desc "Show time stats for a project #{issuable_name}" - params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - end - get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do - authorize! read_issuable_key, load_issuable - - present load_issuable, with: ::API::Entities::IssuableTimeStats - end - end - end - end -end diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb deleted file mode 100644 index 3e2c61f6dbd..00000000000 --- a/lib/api/v3/todos.rb +++ /dev/null @@ -1,30 +0,0 @@ -module API - module V3 - class Todos < Grape::API - before { authenticate! } - - resource :todos do - desc 'Mark a todo as done' do - success ::API::Entities::Todo - end - params do - requires :id, type: Integer, desc: 'The ID of the todo being marked as done' - end - delete ':id' do - TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = current_user.todos.find(params[:id]) - - present todo, with: ::API::Entities::Todo, current_user: current_user - end - - desc 'Mark all todos as done' - delete do - status(200) - - todos = TodosFinder.new(current_user, params).execute - TodoService.new.mark_todos_as_done(todos, current_user).size - end - end - end - end -end diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb deleted file mode 100644 index 34f07dfb486..00000000000 --- a/lib/api/v3/triggers.rb +++ /dev/null @@ -1,112 +0,0 @@ -module API - module V3 - class Triggers < Grape::API - include PaginationParams - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Trigger a GitLab project build' do - success ::API::V3::Entities::TriggerRequest - end - params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' - requires :token, type: String, desc: 'The unique token of trigger' - optional :variables, type: Hash, desc: 'The list of variables to be injected into build' - end - post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42121') - - # validate variables - params[:variables] = params[:variables].to_h - unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - project = find_project(params[:id]) - not_found! unless project - - result = Ci::PipelineTriggerService.new(project, nil, params).execute - not_found! unless result - - if result[:http_status] - render_api_error!(result[:message], result[:http_status]) - else - pipeline = result[:pipeline] - - # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. - # Ci::TriggerRequest doesn't save variables anymore. - # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. - # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. - trigger_request = pipeline.trigger_requests.last - trigger_request.variables = params[:variables] - - present trigger_request, with: ::API::V3::Entities::TriggerRequest - end - end - - desc 'Get triggers list' do - success ::API::V3::Entities::Trigger - end - params do - use :pagination - end - get ':id/triggers' do - authenticate! - authorize! :admin_build, user_project - - triggers = user_project.triggers.includes(:trigger_requests) - - present paginate(triggers), with: ::API::V3::Entities::Trigger - end - - desc 'Get specific trigger of a project' do - success ::API::V3::Entities::Trigger - end - params do - requires :token, type: String, desc: 'The unique token of trigger' - end - get ':id/triggers/:token' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find_by(token: params[:token].to_s) - return not_found!('Trigger') unless trigger - - present trigger, with: ::API::V3::Entities::Trigger - end - - desc 'Create a trigger' do - success ::API::V3::Entities::Trigger - end - post ':id/triggers' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.create - - present trigger, with: ::API::V3::Entities::Trigger - end - - desc 'Delete a trigger' do - success ::API::V3::Entities::Trigger - end - params do - requires :token, type: String, desc: 'The unique token of trigger' - end - delete ':id/triggers/:token' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find_by(token: params[:token].to_s) - return not_found!('Trigger') unless trigger - - trigger.destroy - - present trigger, with: ::API::V3::Entities::Trigger - end - end - end - end -end diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb deleted file mode 100644 index cf106f2552d..00000000000 --- a/lib/api/v3/users.rb +++ /dev/null @@ -1,204 +0,0 @@ -module API - module V3 - class Users < Grape::API - include PaginationParams - include APIGuard - - allow_access_with_scope :read_user, if: -> (request) { request.get? } - - before do - authenticate! - end - - resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do - helpers do - params :optional_attributes do - optional :skype, type: String, desc: 'The Skype username' - optional :linkedin, type: String, desc: 'The LinkedIn username' - optional :twitter, type: String, desc: 'The Twitter username' - optional :website_url, type: String, desc: 'The website of the user' - optional :organization, type: String, desc: 'The organization of the user' - optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' - optional :extern_uid, type: String, desc: 'The external authentication provider UID' - optional :provider, type: String, desc: 'The external provider' - optional :bio, type: String, desc: 'The biography of the user' - optional :location, type: String, desc: 'The location of the user' - optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' - optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' - optional :confirm, type: Boolean, default: true, desc: 'Flag indicating the account needs to be confirmed' - optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' - all_or_none_of :extern_uid, :provider - end - end - - desc 'Create a user. Available only for admins.' do - success ::API::Entities::UserPublic - end - params do - requires :email, type: String, desc: 'The email of the user' - optional :password, type: String, desc: 'The password of the new user' - optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' - at_least_one_of :password, :reset_password - requires :name, type: String, desc: 'The name of the user' - requires :username, type: String, desc: 'The username of the user' - use :optional_attributes - end - post do - authenticated_as_admin! - - params = declared_params(include_missing: false) - user = ::Users::CreateService.new(current_user, params.merge!(skip_confirmation: !params[:confirm])).execute - - if user.persisted? - present user, with: ::API::Entities::UserPublic - else - conflict!('Email has already been taken') if User - .where(email: user.email) - .count > 0 - - conflict!('Username has already been taken') if User - .where(username: user.username) - .count > 0 - - render_validation_error!(user) - end - end - - desc 'Get the SSH keys of a specified user. Available only for admins.' do - success ::API::Entities::SSHKey - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/keys' do - authenticated_as_admin! - - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - present paginate(user.keys), with: ::API::Entities::SSHKey - end - - desc 'Get the emails addresses of a specified user. Available only for admins.' do - success ::API::Entities::Email - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/emails' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - present user.emails, with: ::API::Entities::Email - end - - desc 'Block a user. Available only for admins.' - params do - requires :id, type: Integer, desc: 'The ID of the user' - end - put ':id/block' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - if !user.ldap_blocked? - user.block - else - forbidden!('LDAP blocked users cannot be modified by the API') - end - end - - desc 'Unblock a user. Available only for admins.' - params do - requires :id, type: Integer, desc: 'The ID of the user' - end - put ':id/unblock' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - if user.ldap_blocked? - forbidden!('LDAP blocked users cannot be unblocked by the API') - else - user.activate - end - end - - desc 'Get the contribution events of a specified user' do - detail 'This feature was introduced in GitLab 8.13.' - success ::API::V3::Entities::Event - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/events' do - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - events = user.events - .merge(ProjectsFinder.new(current_user: current_user).execute) - .references(:project) - .with_associations - .recent - - present paginate(events), with: ::API::V3::Entities::Event - end - - desc 'Delete an existing SSH key from a specified user. Available only for admins.' do - success ::API::Entities::SSHKey - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - requires :key_id, type: Integer, desc: 'The ID of the SSH key' - end - delete ':id/keys/:key_id' do - authenticated_as_admin! - - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - key = user.keys.find_by(id: params[:key_id]) - not_found!('Key') unless key - - present key.destroy, with: ::API::Entities::SSHKey - end - end - - resource :user do - desc "Get the currently authenticated user's SSH keys" do - success ::API::Entities::SSHKey - end - params do - use :pagination - end - get "keys" do - present current_user.keys, with: ::API::Entities::SSHKey - end - - desc "Get the currently authenticated user's email addresses" do - success ::API::Entities::Email - end - get "emails" do - present current_user.emails, with: ::API::Entities::Email - end - - desc 'Delete an SSH key from the currently authenticated user' do - success ::API::Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the SSH key' - end - delete "keys/:key_id" do - key = current_user.keys.find_by(id: params[:key_id]) - not_found!('Key') unless key - - present key.destroy, with: ::API::Entities::SSHKey - end - end - end - end -end diff --git a/lib/api/v3/variables.rb b/lib/api/v3/variables.rb deleted file mode 100644 index 83972b1e7ce..00000000000 --- a/lib/api/v3/variables.rb +++ /dev/null @@ -1,29 +0,0 @@ -module API - module V3 - class Variables < Grape::API - include PaginationParams - - before { authenticate! } - before { authorize! :admin_build, user_project } - - params do - requires :id, type: String, desc: 'The ID of a project' - end - - resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Delete an existing variable from a project' do - success ::API::Entities::Variable - end - params do - requires :key, type: String, desc: 'The key of the variable' - end - delete ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key]) - not_found!('Variable') unless variable - - present variable.destroy, with: ::API::Entities::Variable - end - end - end - end -end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index d08876ae1b9..a34de9410e8 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -31,7 +31,7 @@ module API key = params[:key] variable = user_project.variables.find_by(key: key) - return not_found!('Variable') unless variable + break not_found!('Variable') unless variable present variable, with: Entities::Variable end @@ -67,7 +67,7 @@ module API put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) - return not_found!('Variable') unless variable + break not_found!('Variable') unless variable variable_params = declared_params(include_missing: false).except(:key) diff --git a/lib/api/version.rb b/lib/api/version.rb index 9ba576bd828..3b10bfa6a7d 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -6,7 +6,7 @@ module API detail 'This feature was introduced in GitLab 8.13.' end get '/version' do - { version: Gitlab::VERSION, revision: Gitlab::REVISION } + { version: Gitlab::VERSION, revision: Gitlab.revision } end end end diff --git a/lib/backup.rb b/lib/backup.rb new file mode 100644 index 00000000000..e2c62af23ae --- /dev/null +++ b/lib/backup.rb @@ -0,0 +1,3 @@ +module Backup + Error = Class.new(StandardError) +end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 4383124d150..45a935ab352 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Artifacts < Files - def initialize - super('artifacts', JobArtifactUploader.root) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir, 0700) + def initialize(progress) + @progress = progress + + super('artifacts', JobArtifactUploader.root) end end end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 635967f4bd4..adf85ca4719 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Builds < Files - def initialize - super('builds', Settings.gitlab_ci.builds_path) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir, 0700) + def initialize(progress) + @progress = progress + + super('builds', Settings.gitlab_ci.builds_path) end end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 5e6828de597..086ca5986bd 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,9 +2,11 @@ require 'yaml' module Backup class Database + attr_reader :progress attr_reader :config, :db_file_name - def initialize + def initialize(progress) + @progress = progress @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -19,12 +21,12 @@ module Backup dump_pid = case config["adapter"] when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " + progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " + progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. if Gitlab.config.backup.pg_schema @@ -42,7 +44,7 @@ module Backup end report_success(success) - abort 'Backup failed' unless success + raise Backup::Error, 'Backup failed' unless success end def restore @@ -53,12 +55,12 @@ module Backup restore_pid = case config["adapter"] when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " + progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " + progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env spawn('psql', config['database'], in: decompress_rd) end @@ -70,7 +72,7 @@ module Backup end report_success(success) - abort 'Restore failed' unless success + abort Backup::Error, 'Restore failed' unless success end protected @@ -111,9 +113,9 @@ module Backup def report_success(success) if success - $progress.puts '[DONE]'.color(:green) + progress.puts '[DONE]'.color(:green) else - $progress.puts '[FAILED]'.color(:red) + progress.puts '[FAILED]'.color(:red) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 287d591e88d..e287aa1e392 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,7 +1,10 @@ require 'open3' +require_relative 'helper' module Backup class Files + include Backup::Helper + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(name, app_files_dir) @@ -23,33 +26,51 @@ module Backup unless status.zero? puts output - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end - run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end end def restore backup_existing_files_dir - create_files_dir - run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) + end + + def tar + if system(*%w[gtar --version], out: '/dev/null') + # It looks like we can get GNU tar by running 'gtar' + 'gtar' + else + 'tar' + end end def backup_existing_files_dir - timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}") if File.exist?(app_files_dir) - FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + FileUtils.mkdir_p(timestamped_files_path, mode: 0700) + files = Dir.glob(File.join(app_files_dir, "*"), File::FNM_DOTMATCH) - [File.join(app_files_dir, "."), File.join(app_files_dir, "..")] + begin + FileUtils.mv(files, timestamped_files_path) + rescue Errno::EACCES + access_denied_error(app_files_dir) + rescue Errno::EBUSY + resource_busy_error(app_files_dir) + end end end def run_pipeline!(cmd_list, options = {}) status_list = Open3.pipeline(*cmd_list, options) - abort 'Backup failed' unless status_list.compact.all?(&:success?) + raise Backup::Error, 'Backup failed' unless status_list.compact.all?(&:success?) end end end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb new file mode 100644 index 00000000000..54b9ce10b4d --- /dev/null +++ b/lib/backup/helper.rb @@ -0,0 +1,31 @@ +module Backup + module Helper + def access_denied_error(path) + message = <<~EOS + + ### NOTICE ### + As part of restore, the task tried to move existing content from #{path}. + However, it seems that directory contains files/folders that are not owned + by the user #{Gitlab.config.gitlab.user}. To proceed, please move the files + or folders inside #{path} to a secure location so that #{path} is empty and + run restore task again. + + EOS + raise message + end + + def resource_busy_error(path) + message = <<~EOS + + ### NOTICE ### + As part of restore, the task tried to rename `#{path}` before restoring. + This could not be completed, perhaps `#{path}` is a mountpoint? + + To complete the restore, please move the contents of `#{path}` to a + different location and run the restore task again. + + EOS + raise message + end + end +end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 4153467fbee..185ff8ae6bd 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Lfs < Files - def initialize - super('lfs', Settings.lfs.storage_path) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir, 0700) + def initialize(progress) + @progress = progress + + super('lfs', Settings.lfs.storage_path) end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f27ce4d2b2b..a3641505196 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -4,6 +4,12 @@ module Backup FOLDERS_TO_BACKUP = %w[repositories db].freeze FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze + attr_reader :progress + + def initialize(progress) + @progress = progress + end + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -14,14 +20,14 @@ module Backup end # create archive - $progress.print "Creating backup archive: #{tar_file} ... " + progress.print "Creating backup archive: #{tar_file} ... " # Set file permissions on open to prevent chmod races. tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "creating archive #{tar_file} failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end upload @@ -29,11 +35,11 @@ module Backup end def upload - $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " + progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - $progress.puts "skipped".color(:yellow) + progress.puts "skipped".color(:yellow) return end @@ -43,31 +49,31 @@ module Backup multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, storage_class: Gitlab.config.backup.upload.storage_class) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end end def cleanup - $progress.print "Deleting tmp directories ... " + progress.print "Deleting tmp directories ... " backup_contents.each do |dir| next unless File.exist?(File.join(backup_path, dir)) if FileUtils.rm_rf(File.join(backup_path, dir)) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end end end def remove_old # delete backups - $progress.print "Deleting old backups ... " + progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i if keep_time > 0 @@ -88,31 +94,32 @@ module Backup FileUtils.rm(file) removed += 1 rescue => e - $progress.puts "Deleting #{file} failed: #{e.message}".color(:red) + progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end end end - $progress.puts "done. (#{removed} removed)".color(:green) + progress.puts "done. (#{removed} removed)".color(:green) else - $progress.puts "skipping".color(:yellow) + progress.puts "skipping".color(:yellow) end end + # rubocop: disable Metrics/AbcSize def unpack Dir.chdir(backup_path) do # check for existing backups in the backup dir if backup_file_list.empty? - $progress.puts "No backups found in #{backup_path}" - $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + progress.puts "No backups found in #{backup_path}" + progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 elsif backup_file_list.many? && ENV["BACKUP"].nil? - $progress.puts 'Found more than one backup:' + progress.puts 'Found more than one backup:' # print list of available backups - $progress.puts " " + available_timestamps.join("\n ") - $progress.puts 'Please specify which one you want to restore:' - $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + progress.puts " " + available_timestamps.join("\n ") + progress.puts 'Please specify which one you want to restore:' + progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end @@ -123,31 +130,31 @@ module Backup end unless File.exist?(tar_file) - $progress.puts "The backup file #{tar_file} does not exist!" + progress.puts "The backup file #{tar_file} does not exist!" exit 1 end - $progress.print 'Unpacking backup ... ' + progress.print 'Unpacking backup ... ' unless Kernel.system(*%W(tar -xf #{tar_file})) - $progress.puts 'unpacking backup failed'.color(:red) + progress.puts 'unpacking backup failed'.color(:red) exit 1 else - $progress.puts 'done'.color(:green) + progress.puts 'done'.color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - $progress.puts(<<~HEREDOC.color(:red)) + progress.puts(<<~HEREDOC.color(:red)) GitLab version mismatch: Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! Please switch to the following version and try again: version: #{settings[:gitlab_version]} HEREDOC - $progress.puts - $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + progress.puts + progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 end end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 215ded93bfe..542e35a7c7c 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Pages < Files - def initialize - super('pages', Gitlab.config.pages.path) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir, 0700) + def initialize(progress) + @progress = progress + + super('pages', Gitlab.config.pages.path) end end end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 67fe0231087..35821805797 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Registry < Files - def initialize - super('registry', Settings.registry.path) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir, 0700) + def initialize(progress) + @progress = progress + + super('registry', Settings.registry.path) end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6715159a1aa..af762db517c 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,144 +1,221 @@ require 'yaml' +require_relative 'helper' module Backup class Repository - # rubocop:disable Metrics/AbcSize + include Backup::Helper + + attr_reader :progress + + def initialize(progress) + @progress = progress + end + def dump prepare Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = path_to_repo(project) - path_to_project_bundle = path_to_bundle(project) - # Create namespace dir or hashed path if missing if project.hashed_storage?(:repository) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) else FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace end - if empty_repo?(project) + if !empty_repo?(project) + backup_project(project) + progress.puts "[DONE]".color(:green) + else progress.puts "[SKIPPED]".color(:cyan) + end + + wiki = ProjectWiki.new(project) + + if !empty_repo?(wiki) + backup_project(wiki) + progress.puts "[DONE] Wiki".color(:green) else - in_path(path_to_project_repo) do |dir| - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end + progress.puts "[SKIPPED] Wiki".color(:cyan) + end + end + end - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) + def prepare_directories + Gitlab.config.repositories.storages.each do |name, repository_storage| + delete_all_repositories(name, repository_storage) + end + end - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end + def backup_project(project) + gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + backup_project_gitaly(project) + else + backup_project_local(project) end + end - wiki = ProjectWiki.new(project) - path_to_wiki_repo = path_to_repo(wiki) - path_to_wiki_bundle = path_to_bundle(wiki) + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') + end + + def backup_project_gitaly(project) + path_to_project_bundle = path_to_bundle(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .create_bundle(path_to_project_bundle) + end + + def backup_project_local(project) + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + + path_to_project_bundle = path_to_bundle(project) + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) + output, status = Gitlab::Popen.popen(cmd) + progress_warn(project, cmd.join(' '), output) unless status.zero? + end + + def delete_all_repositories(name, repository_storage) + gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories + else + local_delete_all_repositories(name, repository_storage) + end + end + end + + def local_delete_all_repositories(name, repository_storage) + path = repository_storage.legacy_disk_path + return unless File.exist?(path) + + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) + end + end + + def local_restore_custom_hooks(project, dir) + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + + def gitaly_restore_custom_hooks(project, dir) + custom_hooks_path = path_to_tars(project, dir) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .restore_custom_hooks(custom_hooks_path) + end - if File.exist?(path_to_wiki_repo) - progress.print " * #{display_repo_path(wiki)} ... " + def local_backup_custom_hooks(project) + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + break unless File.exist?(File.join(path_to_project_repo, dir)) + + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) - if empty_repo?(wiki) - progress.puts " [SKIPPED]".color(:cyan) + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end + + def gitaly_backup_custom_hooks(project) + FileUtils.mkdir_p(path_to_tars(project)) + custom_hooks_path = path_to_tars(project, 'custom_hooks') + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .backup_custom_hooks(custom_hooks_path) + end + + def backup_custom_hooks(project) + gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + gitaly_backup_custom_hooks(project) + else + local_backup_custom_hooks(project) + end + end + end + + def restore_custom_hooks(project) + in_path(path_to_tars(project)) do |dir| + gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + gitaly_restore_custom_hooks(project, dir) else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(wiki, cmd.join(' '), output) - end + local_restore_custom_hooks(project, dir) end end end end def restore - Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage['path'] - next unless File.exist?(path) - - # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(path, bk_repos_path) - # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 02770) - end + prepare_directories + gitlab_shell = Gitlab::Shell.new Project.find_each(batch_size: 1000) do |project| - progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = path_to_repo(project) + progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exists - cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) - else - %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) - end + restore_repo_success = nil + if File.exist?(path_to_project_bundle) + begin + project.repository.create_from_bundle path_to_project_bundle + restore_repo_success = true + rescue => e + restore_repo_success = false + progress.puts "Error: #{e}".color(:red) + end + else + restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path) + end - output, status = Gitlab::Popen.popen(cmd) - if status.zero? + if restore_repo_success progress.puts "[DONE]".color(:green) else - progress_warn(project, cmd.join(' '), output) + progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - in_path(path_to_tars(project)) do |dir| - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end + restore_custom_hooks(project) wiki = ProjectWiki.new(project) - path_to_wiki_repo = path_to_repo(wiki) path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{display_repo_path(wiki)} ... " - - # If a wiki bundle exists, first remove the empty repo - # that was initialized with ProjectWiki.new() and then - # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_wiki_repo) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) + progress.print " * #{wiki.full_path} ... " + begin + wiki.repository.create_from_bundle(path_to_wiki_bundle) + restore_custom_hooks(wiki) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) + progress.puts "[DONE]".color(:green) + rescue => e + progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) + progress.puts "Error #{e}".color(:red) end end end - - progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - puts " [FAILED]".color(:red) - puts "failed: #{cmd}" - puts output - end end - # rubocop:enable Metrics/AbcSize protected @@ -176,9 +253,7 @@ module Backup def prepare FileUtils.rm_rf(backup_repos_path) - # Ensure the parent dir of backup_repos_path exists FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_repos_path before us FileUtils.mkdir(backup_repos_path, mode: 0700) end @@ -194,21 +269,22 @@ module Backup end def empty_repo?(project_or_wiki) - # Protect against stale caches project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.empty? end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } - end - - def progress - $progress + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + raise Error, e + end end end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 35118375499..49b117a7ee3 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,12 +2,12 @@ require 'backup/files' module Backup class Uploads < Files - def initialize - super('uploads', Rails.root.join('public/uploads')) - end + attr_reader :progress - def create_files_dir - Dir.mkdir(app_files_dir) + def initialize(progress) + @progress = progress + + super('uploads', Rails.root.join('public/uploads')) end end end diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb index f5ff95e3eb3..c351a155ae5 100644 --- a/lib/banzai/commit_renderer.rb +++ b/lib/banzai/commit_renderer.rb @@ -3,7 +3,7 @@ module Banzai ATTRIBUTES = [:description, :title].freeze def self.render(commits, project, user = nil) - obj_renderer = ObjectRenderer.new(project, user) + obj_renderer = ObjectRenderer.new(user: user, default_project: project) ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) } end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index d8fb7705b2a..3f1e95d4cc0 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -4,7 +4,7 @@ module Banzai module CrossProjectReference # Given a cross-project reference string, get the Project record # - # Defaults to value of `context[:project]` if: + # Defaults to value of `context[:project]`, or `context[:group]` if: # * No reference is given OR # * Reference given doesn't exist # @@ -12,7 +12,7 @@ module Banzai # # Returns a Project, or nil if the reference can't be found def parent_from_ref(ref) - return context[:project] unless ref + return context[:project] || context[:group] unless ref Project.find_by_full_path(ref) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index c9e3f8ce42b..60a12dca9d3 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -56,29 +56,29 @@ module Banzai # Implement in child class # Example: project.merge_requests.find - def find_object(project, id) + def find_object(parent_object, id) end # Override if the link reference pattern produces a different ID (global # ID vs internal ID, for instance) to the regular reference pattern. - def find_object_from_link(project, id) - find_object(project, id) + def find_object_from_link(parent_object, id) + find_object(parent_object, id) end # Implement in child class # Example: project_merge_request_url - def url_for_object(object, project) + def url_for_object(object, parent_object) end - def find_object_cached(project, id) - cached_call(:banzai_find_object, id, path: [object_class, project.id]) do - find_object(project, id) + def find_object_cached(parent_object, id) + cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do + find_object(parent_object, id) end end - def find_object_from_link_cached(project, id) - cached_call(:banzai_find_object_from_link, id, path: [object_class, project.id]) do - find_object_from_link(project, id) + def find_object_from_link_cached(parent_object, id) + cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do + find_object_from_link(parent_object, id) end end @@ -88,9 +88,9 @@ module Banzai end end - def url_for_object_cached(object, project) - cached_call(:banzai_url_for_object, object, path: [object_class, project.id]) do - url_for_object(object, project) + def url_for_object_cached(object, parent_object) + cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do + url_for_object(object, parent_object) end end @@ -171,7 +171,7 @@ module Banzai end if object - title = object_link_title(object) + title = object_link_title(object, matches) klass = reference_class(object_sym) data = data_attributes_for(link_content || match, parent, object, @@ -196,13 +196,15 @@ module Banzai end end - def data_attributes_for(text, project, object, link_content: false, link_reference: false) + def data_attributes_for(text, parent, object, link_content: false, link_reference: false) + object_parent_type = parent.is_a?(Group) ? :group : :project + data_attribute( - original: text, - link: link_content, - link_reference: link_reference, - project: project.id, - object_sym => object.id + original: text, + link: link_content, + link_reference: link_reference, + object_parent_type => parent.id, + object_sym => object.id ) end @@ -213,10 +215,14 @@ module Banzai extras << "comment #{$1}" end + extension = matches[:extension] if matches.names.include?("extension") + + extras << extension if extension + extras end - def object_link_title(object) + def object_link_title(object, matches) object.title end @@ -337,6 +343,12 @@ module Banzai def parent parent_type == :project ? project : group end + + def full_group_path(group_ref) + return current_parent_path unless group_ref + + group_ref + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 75b64ae9af2..4a143baeef6 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -21,12 +21,13 @@ module Banzai # # See http://en.wikipedia.org/wiki/URI_scheme # - # The negative lookbehind ensures that users can paste a URL followed by a - # period or comma for punctuation without those characters being included - # in the generated link. + # The negative lookbehind ensures that users can paste a URL followed by + # punctuation without those characters being included in the generated + # link. It matches the behaviour of Rinku 2.0.1: + # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # - # Rubular: http://rubular.com/r/JzPhi6DCZp - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} + # Rubular: http://rubular.com/r/nrL3r9yUiq + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -104,8 +105,12 @@ module Banzai end end - options = link_options.merge(href: match) - content_tag(:a, match.html_safe, options) + dropped + # match has come from node.to_html above, so we know it's encoded + # correctly. + html_safe_match = match.html_safe + options = link_options.merge(href: html_safe_match) + + content_tag(:a, html_safe_match, options) + dropped end def autolink_filter(text) diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index d2c4b1e4d76..fbfcd72c916 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -10,7 +10,7 @@ module Banzai ^``` .+? - \n```$ + \n```\ *$ ) | (?<html> @@ -19,9 +19,9 @@ module Banzai # Anything, including `>>>` blocks which are ignored by this filter # </tag> - ^<[^>]+?>\n + ^<[^>]+?>\ *\n .+? - \n<\/[^>]+?>$ + \n<\/[^>]+?>\ *$ ) | (?: @@ -30,14 +30,14 @@ module Banzai # Anything, including code and HTML blocks # >>> - ^>>>\n + ^>>>\ *\n (?<quote> (?: # Any character that doesn't introduce a code or HTML block (?! ^``` | - ^<[^>]+?>\n + ^<[^>]+?>\ *\n ) . | @@ -48,7 +48,7 @@ module Banzai \g<html> )+? ) - \n>>>$ + \n>>>\ *$ ) }mx.freeze diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 21bcb1c5ca8..01b3b0dafb9 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -23,6 +23,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + range = CommitRange.new(id, project) range.valid_commits? ? range : nil @@ -34,7 +36,7 @@ module Banzai range.to_param.merge(only_path: context[:only_path])) end - def object_link_title(range) + def object_link_title(range, matches) nil end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 43bf4fc6565..8cd92a1adba 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -17,6 +17,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + if project && project.valid_repo? # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb new file mode 100644 index 00000000000..7b55e8b36f6 --- /dev/null +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -0,0 +1,151 @@ +module Banzai + module Filter + # HTML filter that replaces users' names and emails in commit trailers + # with links to their GitLab accounts or mailto links to their mentioned + # emails. + # + # Commit trailers are special labels in the form of `*-by:` and fall on a + # single line, ex: + # + # Reported-By: John S. Doe <john.doe@foo.bar> + # + # More info about this can be found here: + # * https://git.wiki.kernel.org/index.php/CommitMessageConventions + class CommitTrailersFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + include AvatarsHelper + + TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze + AUTHOR_REGEXP = /(?<author_name>.+)/.freeze + # Devise.email_regexp wouldn't work here since its designed to match + # against strings that only contains email addresses; the \A and \z + # around the expression will only match if the string being matched + # contains just the email nothing else. + MAIL_REGEXP = /<(?<author_email>[^@\s]+@[^@\s]+)>/.freeze + FILTER_REGEXP = /(?<trailer>^\s*#{TRAILER_REGEXP}\s*#{AUTHOR_REGEXP}\s+#{MAIL_REGEXP}$)/mi.freeze + + def call + doc.xpath('descendant-or-self::text()').each do |node| + content = node.to_html + + next unless content.match(FILTER_REGEXP) + + html = trailer_filter(content) + + next if html == content + + node.replace(html) + end + + doc + end + + private + + # Replace trailer lines with links to GitLab users or mailto links to + # non GitLab users. + # + # text - String text to replace trailers in. + # + # Returns a String with all trailer lines replaced with links to GitLab + # users and mailto links to non GitLab users. All links have `data-trailer` + # and `data-user` attributes attached. + def trailer_filter(text) + text.gsub(FILTER_REGEXP) do |author_match| + label = $~[:label] + "#{label} #{parse_user($~[:author_name], $~[:author_email], label)}" + end + end + + # Find a GitLab user using the supplied email and generate + # a valid link to them, otherwise, generate a mailto link. + # + # name - String name used in the commit message for the user + # email - String email used in the commit message for the user + # trailer - String trailer used in the commit message + # + # Returns a String with a link to the user. + def parse_user(name, email, trailer) + link_to_user User.find_by_any_email(email), + name: name, + email: email, + trailer: trailer + end + + def urls + Gitlab::Routing.url_helpers + end + + def link_to_user(user, name:, email:, trailer:) + wrapper = link_wrapper(data: { + trailer: trailer, + user: user.try(:id) + }) + + avatar = user_avatar_without_link( + user: user, + user_email: email, + css_class: 'avatar-inline', + has_tooltip: false + ) + + link_href = user.nil? ? "mailto:#{email}" : urls.user_url(user) + + avatar_link = link_tag( + link_href, + content: avatar, + title: email + ) + + name_link = link_tag( + link_href, + content: name, + title: email + ) + + email_link = link_tag( + "mailto:#{email}", + content: email, + title: email + ) + + wrapper << "#{avatar_link}#{name_link} <#{email_link}>" + end + + def link_wrapper(data: {}) + data_attributes = data_attributes_from_hash(data) + + doc.document.create_element( + 'span', + data_attributes + ) + end + + def link_tag(url, title: "", content: "", data: {}) + data_attributes = data_attributes_from_hash(data) + + attributes = data_attributes.merge( + href: url, + title: title + ) + + link = doc.document.create_element('a', attributes) + + if content.html_safe? + link << content + else + link.content = content # make sure we escape content using nokogiri's #content= + end + + link + end + + def data_attributes_from_hash(data = {}) + data.reject! {|_, value| value.nil?} + data.map do |key, value| + [%(data-#{key.to_s.dasherize}), value] + end.to_h + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index b82c6ca6393..e1261e7bbbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -11,7 +11,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| content = node.to_html next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index c2b42673376..bb9f488cd87 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -56,8 +56,13 @@ module Banzai # Pattern to match allowed image extensions ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze + # Do not perform linking inside these tags. + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it # needs special-case handling diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index beb21b19ab3..73e82a4d7e3 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -4,7 +4,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) content = node.to_html diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 77299abe324..1a415232545 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -11,13 +11,14 @@ module Banzai def call return doc unless context[:issuable_state_filter_enabled] - extractor = Banzai::IssuableExtractor.new(project, current_user) + context = RenderContext.new(project, current_user) + extractor = Banzai::IssuableExtractor.new(context) issuables = extractor.extract([doc]) issuables.each do |node, issuable| next if !can_read_cross_project? && issuable.project != project - if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" end end @@ -27,6 +28,10 @@ module Banzai private + def issuable_reference?(text, issuable) + text == issuable.reference_link_text(project || group) + end + def can_read_cross_project? Ability.allowed?(current_user, :read_cross_project) end @@ -38,6 +43,10 @@ module Banzai def project context[:project] end + + def group + context[:group] + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index d5360ad8f68..a5f38046a43 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -8,8 +8,8 @@ module Banzai Label end - def find_object(project, id) - find_labels(project).find(id) + def find_object(parent_object, id) + find_labels(parent_object).find(id) end def self.references_in(text, pattern = Label.reference_pattern) @@ -32,16 +32,25 @@ module Banzai end end - def find_label(project_ref, label_id, label_name) - project = parent_from_ref(project_ref) - return unless project + def find_label(parent_ref, label_id, label_name) + parent = parent_from_ref(parent_ref) + return unless parent label_params = label_params(label_id, label_name) - find_labels(project).find_by(label_params) + find_labels(parent).find_by(label_params) end - def find_labels(project) - LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) + def find_labels(parent) + params = if parent.is_a?(Group) + { group_id: parent.id, + include_ancestor_groups: true, + only_group_labels: true } + else + { project_id: parent.id, + include_ancestor_groups: true } + end + + LabelsFinder.new(nil, params).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -59,25 +68,39 @@ module Banzai end end - def url_for_object(label, project) + def url_for_object(label, parent) h = Gitlab::Routing.url_helpers - h.project_issues_url(project, label_name: label.name, only_path: context[:only_path]) + + if parent.is_a?(Project) + h.project_issues_url(parent, label_name: label.name, only_path: context[:only_path]) + elsif context[:label_url_method] + h.public_send(context[:label_url_method], parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend + end end def object_link_text(object, matches) - project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = from_ref_cached(project_path) - reference = project_from_ref.to_human_reference(project) - label_suffix = " <i>in #{reference}</i>" if reference.present? + label_suffix = '' + + if project || full_path_ref?(matches) + project_path = full_project_path(matches[:namespace], matches[:project]) + parent_from_ref = from_ref_cached(project_path) + reference = parent_from_ref.to_human_reference(project || group) + + label_suffix = " <i>in #{reference}</i>" if reference.present? + end LabelsHelper.render_colored_label(object, label_suffix) end + def full_path_ref?(matches) + matches[:namespace] && matches[:project] + end + def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end - def object_link_title(object) + def object_link_title(object, matches) # use title of wrapped element instead nil end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index c1e2b680240..944363f17d3 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -14,7 +14,7 @@ module Banzai private - DEFAULT_ENGINE = :redcarpet + DEFAULT_ENGINE = :common_mark def engine(engine_from_context) engine_from_context ||= DEFAULT_ENGINE diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index b3cfa97d0e0..5cbdb01c130 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -17,10 +17,19 @@ module Banzai only_path: context[:only_path]) end + def object_link_title(object, matches) + object_link_commit_title(object, matches) || super + end + def object_link_text_extras(object, matches) extras = super + if commit_ref = object_link_commit_ref(object, matches) + return extras.unshift(commit_ref) + end + path = matches[:path] if matches.names.include?("path") + case path when '/diffs' extras.unshift "diffs" @@ -38,6 +47,36 @@ module Banzai .where(iid: ids.to_a) .includes(target_project: :namespace) end + + private + + def object_link_commit_title(object, matches) + object_link_commit(object, matches)&.title + end + + def object_link_commit_ref(object, matches) + object_link_commit(object, matches)&.short_id + end + + def object_link_commit(object, matches) + return unless matches.names.include?('query') && query = matches[:query] + + # Removes leading "?". CGI.parse expects "arg1&arg2&arg3" + params = CGI.parse(query.sub(/^\?/, '')) + + return unless commit_sha = params['commit_id']&.first + + if commit = find_commit_by_sha(object, commit_sha) + Commit.from_hash(commit.to_hash, object.project) + end + end + + def find_commit_by_sha(object, commit_sha) + @all_commits ||= {} + @all_commits[object.id] ||= object.all_commits + + @all_commits[object.id].find { |commit| commit.sha == commit_sha } + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 8ec696ce5fc..af8448937b3 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -12,10 +12,14 @@ module Banzai # 'regular' references, we need to use the global ID to disambiguate # between group and project milestones. def find_object(project, id) + return unless project.is_a?(Project) + find_milestone_with_finder(project, id: id) end def find_object_from_link(project, iid) + return unless project.is_a?(Project) + find_milestone_with_finder(project, iid: iid) end @@ -40,7 +44,7 @@ module Banzai project_path = full_project_path(namespace_ref, project_ref) project = parent_from_ref(project_path) - return unless project + return unless project && project.is_a?(Project) milestone_params = milestone_params(milestone_id, milestone_name) @@ -61,7 +65,7 @@ module Banzai # We don't support IID lookups for group milestones, because IIDs can # clash between group and project milestones. if project.group && !params[:iid] - finder_params[:group_ids] = [project.group.id] + finder_params[:group_ids] = project.group.self_and_ancestors_ids end MilestonesFinder.new(finder_params).find_by(params) @@ -84,7 +88,7 @@ module Banzai end end - def object_link_title(object) + def object_link_title(object, matches) nil end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 5325819d828..28933c78966 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -23,7 +23,7 @@ module Banzai private def settings - ApplicationSetting.current || ApplicationSetting.create_from_defaults + Gitlab::CurrentSettings.current_application_settings end def plantuml_setup diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 9f9882b3b40..caf11fe94c4 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,7 +7,11 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction] + unless context[:skip_redaction] + context = RenderContext.new(project, current_user) + + Redactor.new(context).redact([doc]) + end doc end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index b9d5ecf70ec..2f023f4f242 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -73,7 +73,7 @@ module Banzai # # Note that while the key might exist, its value could be nil! def validate - needs :project + needs :project unless skip_project_check? end # Iterates over all <a> and text() nodes in a document. diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6786b9d07b6..afc2ca4e362 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,10 +25,11 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow table alignment; we whitelist specific style properties in a + # Allow table alignment; we whitelist specific text-align values in a # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) + whitelist[:css] = { properties: ['text-align'] } # Allow span elements whitelist[:elements].push('span') diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 134a192c22b..881e10afb9f 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -12,6 +12,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + project.snippets.find_by(id: id) end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 97244159985..b32660a8341 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -92,7 +92,7 @@ module Banzai def text return '' unless node - @text ||= node.text + @text ||= EscapeUtils.escape_html(node.text) end private diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 49603d0b363..ae7dc71e7eb 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -12,11 +12,11 @@ module Banzai [@data-reference-type="issue" or @data-reference-type="merge_request"] ).freeze - attr_reader :project, :user + attr_reader :context - def initialize(project, user) - @project = project - @user = user + # context - An instance of Banzai::RenderContext. + def initialize(context) + @context = context end # Returns Hash in the form { node => issuable_instance } @@ -25,8 +25,10 @@ module Banzai document.xpath(QUERY) end - issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) - merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) + issue_parser = Banzai::ReferenceParser::IssueParser.new(context) + + merge_request_parser = + Banzai::ReferenceParser::MergeRequestParser.new(context) issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( merge_request_parser.records_for_nodes(nodes) diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 2691be81623..a176f1e261b 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -13,14 +13,13 @@ module Banzai # As an example, rendering the attribute `note` would place the unredacted # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer - attr_reader :project, :user + attr_reader :context - # project - A Project to use for redacting Markdown. + # default_project - A default Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. # redaction_context - A Hash containing extra attributes to use during redaction - def initialize(project, user = nil, redaction_context = {}) - @project = project - @user = user + def initialize(default_project: nil, user: nil, redaction_context: {}) + @context = RenderContext.new(default_project, user) @redaction_context = base_context.merge(redaction_context) end @@ -48,17 +47,21 @@ module Banzai pipeline = HTML::Pipeline.new([]) objects.map do |object| - pipeline.to_document(Banzai.render_field(object, attribute)) + document = pipeline.to_document(Banzai.render_field(object, attribute)) + + context.associate_document(document, object) + + document end end def post_process_documents(documents, objects, attribute) # Called here to populate cache, refer to IssuableExtractor docs - IssuableExtractor.new(project, user).extract(documents) + IssuableExtractor.new(context).extract(documents) documents.zip(objects).map do |document, object| - context = context_for(object, attribute) - Banzai::Pipeline[:post_process].to_document(document, context) + pipeline_context = context_for(document, object, attribute) + Banzai::Pipeline[:post_process].to_document(document, pipeline_context) end end @@ -66,20 +69,21 @@ module Banzai # # Returns an Array containing the redacted documents. def redact_documents(documents) - redactor = Redactor.new(project, user) + redactor = Redactor.new(context) redactor.redact(documents) end # Returns a Banzai context for the given object and attribute. - def context_for(object, attribute) - @redaction_context.merge(object.banzai_render_context(attribute)) + def context_for(document, object, attribute) + @redaction_context.merge(object.banzai_render_context(attribute)).merge( + project: context.project_for_node(document) + ) end def base_context { - current_user: user, - project: project, + current_user: context.current_user, skip_redaction: true } end diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb new file mode 100644 index 00000000000..607c2731ed3 --- /dev/null +++ b/lib/banzai/pipeline/commit_description_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class CommitDescriptionPipeline < SingleLinePipeline + def self.filters + @filters ||= super.concat FilterArray[ + Filter::CommitTrailersFilter, + ] + end + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8b2f05fffec..a1f24e8b093 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -42,9 +42,9 @@ module Banzai end def self.transform_context(context) - context.merge( - only_path: true, + context[:only_path] = true unless context.key?(:only_path) + context.merge( # EmojiFilter asset_host: Gitlab::Application.config.asset_host, asset_root: Gitlab.config.gitlab.base_url diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index fd457bebf03..28928d6f376 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -2,13 +2,15 @@ module Banzai # Class for removing Markdown references a certain user is not allowed to # view. class Redactor - attr_reader :user, :project + attr_reader :context - # project - A Project to use for redacting links. - # user - The currently logged in user (if any). - def initialize(project, user = nil) - @project = project - @user = user + # context - An instance of `Banzai::RenderContext`. + def initialize(context) + @context = context + end + + def user + context.current_user end # Redacts the references in the given Array of documents. @@ -70,11 +72,11 @@ module Banzai end def redact_cross_project_references(documents) - extractor = Banzai::IssuableExtractor.new(project, user) + extractor = Banzai::IssuableExtractor.new(context) issuables = extractor.extract(documents) issuables.each do |node, issuable| - next if issuable.project == project + next if issuable.project == context.project_for_node(node) node['class'] = node['class'].gsub('has-tooltip', '') node['title'] = nil @@ -95,7 +97,7 @@ module Banzai end per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, user) + parser = Banzai::ReferenceParser[type].new(context) visible.merge(parser.nodes_visible_to_user(user, nodes)) end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 7e6357f8a00..78588299c18 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -10,8 +10,8 @@ module Banzai end def references(type, project, current_user = nil) - processor = Banzai::ReferenceParser[type] - .new(project, current_user) + context = RenderContext.new(project, current_user) + processor = Banzai::ReferenceParser[type].new(context) processor.process(html_documents) end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 279fca8d043..68752f5bb5a 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -45,9 +45,13 @@ module Banzai @data_attribute ||= "data-#{reference_type.to_s.dasherize}" end - def initialize(project = nil, current_user = nil) - @project = project - @current_user = current_user + # context - An instance of `Banzai::RenderContext`. + def initialize(context) + @context = context + end + + def project_for_node(node) + context.project_for_node(node) end # Returns all the nodes containing references that the user can refer to. @@ -224,7 +228,11 @@ module Banzai private - attr_reader :current_user, :project + attr_reader :context + + def current_user + context.current_user + end # When a feature is disabled or visible only for # team members we should not allow team members diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index a50e6f8ef8f..2920e886938 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -29,6 +29,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + range = CommitRange.new(id, project) range.valid_commits? ? range : nil diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 230827129b6..7b5915899cf 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -5,15 +5,10 @@ module Banzai def nodes_visible_to_user(user, nodes) issues = records_for_nodes(nodes) - issues_to_check = issues.values + issues_to_check, cross_project_issues = partition_issues(issues, user) - unless can?(user, :read_cross_project) - issues_to_check, cross_project_issues = issues_to_check.partition do |issue| - issue.project == project - end - end - - readable_issues = Ability.issues_readable_by_user(issues_to_check, user).to_set + readable_issues = + Ability.issues_readable_by_user(issues_to_check, user).to_set nodes.select do |node| issue_in_node = issues[node] @@ -25,7 +20,7 @@ module Banzai # but not the issue. if readable_issues.include?(issue_in_node) true - elsif cross_project_issues&.include?(issue_in_node) + elsif cross_project_issues.include?(issue_in_node) can_read_reference?(user, issue_in_node) else false @@ -33,6 +28,32 @@ module Banzai end end + # issues - A Hash mapping HTML nodes to their corresponding Issue + # instances. + # user - The current User. + def partition_issues(issues, user) + return [issues.values, []] if can?(user, :read_cross_project) + + issues_to_check = [] + cross_project_issues = [] + + # We manually partition the data since our input is a Hash and our + # output has to be an Array of issues; not an Array of (node, issue) + # pairs. + issues.each do |node, issue| + target = + if issue.project == project_for_node(node) + issues_to_check + else + cross_project_issues + end + + target << issue + end + + [issues_to_check, cross_project_issues] + end + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, @@ -48,7 +69,8 @@ module Banzai { group: [:owners, :group_members] }, :invited_groups, :project_members, - :project_feature + :project_feature, + :route ] } ), diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 8932d4f2905..ceb7f1d165c 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -58,7 +58,7 @@ module Banzai def can_read_project_reference?(node) node_id = node.attr('data-project').to_i - project && project.id == node_id + project_for_node(node)&.id == node_id end def nodes_user_can_reference(current_user, nodes) @@ -71,6 +71,7 @@ module Banzai nodes.select do |node| project_id = node.attr(project_attr) user_id = node.attr(author_attr) + project = project_for_node(node) if project && project_id && project.id == project_id.to_i true diff --git a/lib/banzai/render_context.rb b/lib/banzai/render_context.rb new file mode 100644 index 00000000000..e30fc9f469b --- /dev/null +++ b/lib/banzai/render_context.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Banzai + # Object storing the current user, project, and other details used when + # parsing Markdown references. + class RenderContext + attr_reader :current_user + + # default_project - The default project to use for all documents, if any. + # current_user - The user viewing the document, if any. + def initialize(default_project = nil, current_user = nil) + @current_user = current_user + @projects = Hash.new(default_project) + end + + # Associates an HTML document with a Project. + # + # document - The HTML document to map to a Project. + # object - The object that produced the HTML document. + def associate_document(document, object) + # XML nodes respond to "document" but will return a Document instance, + # even when they belong to a DocumentFragment. + document = document.document if document.fragment? + + @projects[document] = object.project if object.respond_to?(:project) + end + + def project_for_node(node) + @projects[node.document] + end + end +end diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb index c7a54629f31..46b609c36b0 100644 --- a/lib/banzai/renderer/common_mark/html.rb +++ b/lib/banzai/renderer/common_mark/html.rb @@ -9,7 +9,7 @@ module Banzai lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' result = "<pre>" \ - "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ "</pre>" out(result) diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb index 94df5d8b1e1..30e815f1224 100644 --- a/lib/banzai/renderer/redcarpet/html.rb +++ b/lib/banzai/renderer/redcarpet/html.rb @@ -6,7 +6,7 @@ module Banzai lang_attr = lang ? %Q{ lang="#{lang}"} : '' "\n<pre>" \ - "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ "</pre>" end end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 054064395c3..44bcbc250b3 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -12,7 +12,7 @@ module Bitbucket end def author - raw.fetch('reporter', {}).fetch('username', nil) + raw.dig('reporter', 'username') end def description diff --git a/lib/constraints/feature_constrainer.rb b/lib/constraints/feature_constrainer.rb new file mode 100644 index 00000000000..05d48b0f25a --- /dev/null +++ b/lib/constraints/feature_constrainer.rb @@ -0,0 +1,13 @@ +module Constraints + class FeatureConstrainer + attr_reader :feature + + def initialize(feature) + @feature = feature + end + + def matches?(_request) + Feature.enabled?(feature) + end + end +end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 77c91817382..87f14b3b0d2 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -77,7 +77,7 @@ module DeclarativePolicy @state = State.new steps_by_score do |step, score| - return if !debug && @state.prevented? + break if !debug && @state.prevented? passed = nil case step.action diff --git a/lib/feature.rb b/lib/feature.rb index 8e9ba5c530a..314ae224d90 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -1,3 +1,6 @@ +require 'flipper/adapters/active_record' +require 'flipper/adapters/active_support_cache_store' + class Feature # Classes to override flipper table names class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature @@ -60,7 +63,15 @@ class Feature end def flipper - @flipper ||= Flipper.instance + if RequestStore.active? + RequestStore[:flipper] ||= build_flipper_instance + else + @flipper ||= build_flipper_instance + end + end + + def build_flipper_instance + Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true } end # This method is called from config/initializers/flipper.rb and can be used @@ -68,5 +79,16 @@ class Feature # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups def register_feature_groups end + + def flipper_adapter + active_record_adapter = Flipper::Adapters::ActiveRecord.new( + feature_class: FlipperFeature, + gate_class: FlipperGate) + + Flipper::Adapters::ActiveSupportCacheStore.new( + active_record_adapter, + Rails.cache, + expires_in: 1.hour) + end end end diff --git a/lib/forever.rb b/lib/forever.rb new file mode 100644 index 00000000000..7df17912544 --- /dev/null +++ b/lib/forever.rb @@ -0,0 +1,13 @@ +class Forever + POSTGRESQL_DATE = DateTime.new(3000, 1, 1) + MYSQL_DATE = DateTime.new(2038, 01, 19) + + # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC + def self.date + if Gitlab::Database.postgresql? + POSTGRESQL_DATE + else + MYSQL_DATE + end + end +end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index aa9fd36d9ff..b9a148f35bf 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,15 +1,54 @@ -require_dependency 'gitlab/git' +require_dependency 'gitlab/popen' module Gitlab + def self.root + Pathname.new(File.expand_path('..', __dir__)) + end + + def self.config + Settings + end + + def self.migrations_hash + @_migrations_hash ||= Digest::MD5.hexdigest(ActiveRecord::Migrator.get_all_versions.to_s) + end + + def self.revision + @_revision ||= begin + if File.exist?(root.join("REVISION")) + File.read(root.join("REVISION")).strip.freeze + else + result = Gitlab::Popen.popen_with_detail(%W[#{config.git.bin_path} log --pretty=format:%h -n 1]) + + if result.status.success? + result.stdout.chomp.freeze + else + "Unknown".freeze + end + end + end + end + COM_URL = 'https://gitlab.com'.freeze APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} + SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} + VERSION = File.read(root.join("VERSION")).strip.freeze + INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze def self.com? - # Check `staging?` as well to keep parity with gitlab.com - Gitlab.config.gitlab.url == COM_URL || staging? + # Check `gl_subdomain?` as well to keep parity with gitlab.com + Gitlab.config.gitlab.url == COM_URL || gl_subdomain? + end + + def self.org? + Gitlab.config.gitlab.url == 'https://dev.gitlab.org' + end + + def self.gl_subdomain? + SUBDOMAIN_REGEX === Gitlab.config.gitlab.url end - def self.staging? - Gitlab.config.gitlab.url == 'https://staging.gitlab.com' + def self.dev_env_or_com? + Rails.env.development? || org? || com? end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 7127948cf00..87e377de4d3 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,10 +29,10 @@ module Gitlab def options { - "Guest" => GUEST, - "Reporter" => REPORTER, - "Developer" => DEVELOPER, - "Master" => MASTER + "Guest" => GUEST, + "Reporter" => REPORTER, + "Developer" => DEVELOPER, + "Maintainer" => MASTER } end @@ -57,10 +57,10 @@ module Gitlab def protection_options { - "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, - "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL + "Not protected: Both developers and maintainers can push new commits, force push, or delete the branch." => PROTECTION_NONE, + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch." => PROTECTION_DEV_CAN_MERGE, + "Partially protected: Both developers and maintainers can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but maintainers can. No-one can force push or delete the branch." => PROTECTION_FULL } end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index f5ccf952cf9..7de66539848 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -5,7 +5,7 @@ module Gitlab REGISTRY_SCOPES = [:read_registry].freeze # Scopes used for GitLab API access - API_SCOPES = [:api, :read_user, :sudo].freeze + API_SCOPES = [:api, :read_user, :sudo, :read_repository].freeze # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze @@ -14,6 +14,25 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self + def omniauth_customized_providers + @omniauth_customized_providers ||= %w[bitbucket jwt] + end + + def omniauth_setup_providers(provider_names) + provider_names.each do |provider| + omniauth_setup_a_provider(provider) + end + end + + def omniauth_setup_a_provider(provider) + case provider + when 'kerberos' + require 'omniauth-kerberos' + when *omniauth_customized_providers + require_dependency "omni_auth/strategies/#{provider}" + end + end + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -26,6 +45,7 @@ module Gitlab lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || + deploy_token_check(login, password) || user_with_password_for_git(login, password) || Gitlab::Auth::Result.new @@ -50,7 +70,7 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - return if user && !user.active? + break if user && !user.active? authenticators = [] @@ -69,7 +89,11 @@ module Gitlab authenticators.compact! - user if authenticators.find { |auth| auth.login(login, password) } + # return found user that was authenticated first for given login credentials + authenticators.find do |auth| + authenticated_user = auth.login(login, password) + break authenticated_user if authenticated_user + end end end @@ -159,7 +183,8 @@ module Gitlab def abilities_for_scopes(scopes) abilities_by_scope = { api: full_authentication_abilities, - read_registry: [:read_container_image] + read_registry: [:read_container_image], + read_repository: [:download_code] } scopes.flat_map do |scope| @@ -167,6 +192,22 @@ module Gitlab end.uniq end + def deploy_token_check(login, password) + return unless password.present? + + token = + DeployToken.active.find_by(token: password) + + return unless token && login + return if login != token.username + + scopes = abilities_for_scopes(token.scopes) + + if valid_scoped_token?(token, available_scopes) + Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) + end + end + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) @@ -199,7 +240,7 @@ module Gitlab return unless login == 'gitlab-ci-token' return unless password - build = ::Ci::Build.running.find_by_token(password) + build = find_build_by_token(password) return unless build return unless build.project.builds_enabled? @@ -260,6 +301,12 @@ module Gitlab REGISTRY_SCOPES end + + private + + def find_build_by_token(token) + ::Ci::Build.running.find_by_token(token) + end end end end diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb index dae03a179e4..7609a7b04f6 100644 --- a/lib/gitlab/auth/blocked_user_tracker.rb +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -17,7 +17,9 @@ module Gitlab # message passed along by Warden. return unless message == User::BLOCKED_MESSAGE - login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') + # Check for either LDAP or regular GitLab account logins + login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') || + env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') return unless login.present? diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index 260a77058a4..1234ace0334 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -8,7 +8,7 @@ module Gitlab def login(login, password) return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user&.valid_password?(password) + return user if user&.valid_password?(password) end end end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 77c0ddc2d48..865185eb5db 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -6,7 +6,7 @@ module Gitlab module Auth module LDAP class Access - attr_reader :provider, :user + attr_reader :provider, :user, :ldap_identity def self.open(user, &block) Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -14,9 +14,12 @@ module Gitlab end end - def self.allowed?(user) + def self.allowed?(user, options = {}) self.open(user) do |access| + # Whether user is allowed, or not, we should update + # permissions to keep things clean if access.allowed? + access.update_user Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true @@ -29,7 +32,8 @@ module Gitlab def initialize(user, adapter = nil) @adapter = adapter @user = user - @provider = user.ldap_identity.provider + @ldap_identity = user.ldap_identity + @provider = adapter&.provider || ldap_identity&.provider end def allowed? @@ -40,7 +44,7 @@ module Gitlab end # Block user in GitLab if he/she was blocked in AD - if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) + if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(ldap_identity.extern_uid, adapter) block_user(user, 'is disabled in Active Directory') false else @@ -52,6 +56,8 @@ module Gitlab block_user(user, 'does not exist anymore') false end + rescue LDAPConnectionError + false end def adapter @@ -62,27 +68,44 @@ module Gitlab Gitlab::Auth::LDAP::Config.new(provider) end + def find_ldap_user + Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter) + end + def ldap_user - @ldap_user ||= Gitlab::Auth::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + return unless provider + + @ldap_user ||= find_ldap_user end def block_user(user, reason) user.ldap_block - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" - ) + if provider + Gitlab::AppLogger.info( + "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + else + Gitlab::AppLogger.info( + "Account is not provided by LDAP, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end def unblock_user(user, reason) user.activate Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ "unblocking Gitlab user \"#{user.name}\" (#{user.email})" ) end + + def update_user + # no-op in CE + end end end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index caf2d18c668..82ff1e77e5c 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -2,6 +2,9 @@ module Gitlab module Auth module LDAP class Adapter + SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + attr_reader :provider, :ldap def self.open(provider, &block) @@ -16,7 +19,7 @@ module Gitlab def initialize(provider, ldap = nil) @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) + @ldap = ldap || renew_connection_adapter end def config @@ -47,8 +50,10 @@ module Gitlab end def ldap_search(*args) + retries ||= 0 + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do + Timeout.timeout(timeout_time(retries)) do results = ldap.search(*args) if results.nil? @@ -63,16 +68,26 @@ module Gitlab results end end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] + rescue Net::LDAP::Error, Timeout::Error => error + retries += 1 + error_message = connection_error_message(error) + + Rails.logger.warn(error_message) + + if retries < MAX_SEARCH_RETRIES + renew_connection_adapter + retry + else + raise LDAPConnectionError, error_message + end end private + def timeout_time(retry_number) + SEARCH_RETRY_FACTOR[retry_number] * config.timeout + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), @@ -104,6 +119,18 @@ module Gitlab filter end end + + def connection_error_message(exception) + if exception.is_a?(Timeout::Error) + "LDAP search timed out after #{config.timeout} seconds" + else + "LDAP search raised exception #{exception.class}: #{exception.message}" + end + end + + def renew_connection_adapter + @ldap = Net::LDAP.new(config.adapter_options) + end end end end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index e70c3ab6b46..7c134fb6438 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -12,30 +12,26 @@ module Gitlab return unless Gitlab::Auth::LDAP::Config.enabled? return unless login.present? && password.present? - auth = nil - # loop through providers until valid bind + # return found user that was authenticated by first provider for given login credentials providers.find do |provider| auth = new(provider) - auth.login(login, password) # true will exit the loop + break auth.user if auth.login(login, password) # true will exit the loop end - - # If (login, password) was invalid for all providers, the value of auth is now the last - # Gitlab::Auth::LDAP::Authentication instance we tried. - auth.user end def self.providers Gitlab::Auth::LDAP::Config.providers end - attr_accessor :ldap_user - def login(login, password) - @ldap_user = adapter.bind_as( + result = adapter.bind_as( filter: user_filter(login), size: 1, password: password ) + return unless result + + @user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(result.dn, provider) end def adapter @@ -56,12 +52,6 @@ module Gitlab filter end - - def user - return unless ldap_user - - Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) - end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 77185f52ced..d4415eaa6dc 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -11,6 +11,8 @@ module Gitlab attr_accessor :provider, :options + InvalidProvider = Class.new(StandardError) + def self.enabled? Gitlab.config.ldap.enabled end @@ -22,6 +24,10 @@ module Gitlab def self.available_servers return [] unless enabled? + _available_servers + end + + def self._available_servers Array.wrap(servers.first) end @@ -34,7 +40,7 @@ module Gitlab end def self.invalid_provider(provider) - raise "Unknown provider (#{provider}). Available providers: #{providers}" + raise InvalidProvider.new("Unknown provider (#{provider}). Available providers: #{providers}") end def initialize(provider) @@ -84,13 +90,17 @@ module Gitlab end def base - options['base'] + @base ||= Person.normalize_dn(options['base']) end def uid options['uid'] end + def label + options['label'] + end + def sync_ssh_keys? sync_ssh_keys.present? end @@ -132,6 +142,10 @@ module Gitlab options['timeout'].to_i end + def external_groups + options['external_groups'] || [] + end + def has_auth? options['password'] || options['bind_dn'] end diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb new file mode 100644 index 00000000000..ef0a695742b --- /dev/null +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -0,0 +1,7 @@ +module Gitlab + module Auth + module LDAP + LDAPConnectionError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 068212d9a21..922d0567d99 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -8,6 +8,8 @@ module Gitlab module Auth module LDAP class User < Gitlab::Auth::OAuth::User + extend ::Gitlab::Utils::Override + class << self def find_by_uid_and_provider(uid, provider) identity = ::Identity.with_extern_uid(provider, uid).take @@ -29,7 +31,8 @@ module Gitlab self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def changed? + override :should_save? + def should_save? gl_user.changed? || gl_user.identities.any?(&:changed?) end @@ -41,6 +44,10 @@ module Gitlab Gitlab::Auth::LDAP::Access.allowed?(gl_user) end + def valid_sign_in? + allowed? && super + end + def ldap_config Gitlab::Auth::LDAP::Config.new(auth_hash.provider) end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb index ed03b9f8b40..d4e7f35c857 100644 --- a/lib/gitlab/auth/o_auth/authentication.rb +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -12,6 +12,7 @@ module Gitlab @user = user end + # Implementation must return user object if login successful def login(login, password) raise NotImplementedError end diff --git a/lib/gitlab/auth/o_auth/identity_linker.rb b/lib/gitlab/auth/o_auth/identity_linker.rb new file mode 100644 index 00000000000..de92d7a214d --- /dev/null +++ b/lib/gitlab/auth/o_auth/identity_linker.rb @@ -0,0 +1,8 @@ +module Gitlab + module Auth + module OAuth + class IdentityLinker < OmniauthIdentityLinkerBase + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index b6a96081278..e7283b2f9e8 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -30,6 +30,10 @@ module Gitlab gl_user.try(:valid?) end + def valid_sign_in? + valid? && persisted? + end + def save(provider = 'OAuth') raise SigninDisabledForProviderError if oauth_provider_disabled? raise SignupDisabledError unless gl_user @@ -64,8 +68,22 @@ module Gitlab user end + def find_and_update! + save if should_save? + + gl_user + end + + def bypass_two_factor? + false + end + protected + def should_save? + true + end + def add_or_update_user_identities return unless gl_user @@ -124,6 +142,9 @@ module Gitlab Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + + rescue Gitlab::Auth::LDAP::LDAPConnectionError + nil end def ldap_config diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb new file mode 100644 index 00000000000..f79ce6bb809 --- /dev/null +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -0,0 +1,51 @@ +module Gitlab + module Auth + class OmniauthIdentityLinkerBase + attr_reader :current_user, :oauth + + def initialize(current_user, oauth) + @current_user = current_user + @oauth = oauth + @changed = false + end + + def link + save if identity.new_record? + end + + def changed? + @changed + end + + def failed? + error_message.present? + end + + def error_message + identity.validate + + identity.errors.full_messages.join(', ') + end + + private + + def save + @changed = identity.save + end + + def identity + @identity ||= current_user.identities + .with_extern_uid(provider, uid) + .first_or_initialize(extern_uid: uid) + end + + def provider + oauth['provider'] + end + + def uid + oauth['uid'] + end + end + end +end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index a0b5cd868c3..66de52506ce 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -16,7 +16,7 @@ module Gitlab end def find_sessionless_user - find_user_from_access_token || find_user_from_rss_token + find_user_from_access_token || find_user_from_feed_token rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index c345a7e3f6c..3bc5e2864df 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -6,6 +6,17 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) end + def authn_context + response_object = auth_hash.extra[:response_object] + return nil if response_object.blank? + + document = response_object.decrypted_document + document ||= response_object.document + return nil if document.blank? + + extract_authn_context(document) + end + private def get_raw(key) @@ -13,6 +24,10 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end + + def extract_authn_context(document) + REXML::XPath.first(document, "//saml:AuthnStatement/saml:AuthnContext/saml:AuthnContextClassRef/text()").to_s + end end end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 2760b1a3247..625dab7c6f4 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::Auth::OAuth::Provider.config_for('saml') end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end + def groups options[:groups_attribute] end @@ -14,6 +18,10 @@ module Gitlab def external_groups options[:external_groups] end + + def admin_groups + options[:admin_groups] + end end end end diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb new file mode 100644 index 00000000000..7e4b191d512 --- /dev/null +++ b/lib/gitlab/auth/saml/identity_linker.rb @@ -0,0 +1,8 @@ +module Gitlab + module Auth + module Saml + class IdentityLinker < OmniauthIdentityLinkerBase + end + end + end +end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index d4024e9ec39..6c3b75f3eb0 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -7,6 +7,8 @@ module Gitlab module Auth module Saml class User < Gitlab::Auth::OAuth::User + extend ::Gitlab::Utils::Override + def save super('SAML') end @@ -18,29 +20,36 @@ module Gitlab user ||= find_or_build_ldap_user if auto_link_ldap_user? user ||= build_new_user if signup_enabled? - if external_users_enabled? && user - # Check if there is overlap between the user's groups and the external groups - # setting then set user as external or internal. - user.external = !(auth_hash.groups & Gitlab::Auth::Saml::Config.external_groups).empty? + if user + user.external = !(auth_hash.groups & saml_config.external_groups).empty? if external_users_enabled? end user end - def changed? + override :should_save? + def should_save? return true unless gl_user gl_user.changed? || gl_user.identities.any?(&:changed?) end + def bypass_two_factor? + saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) + end + protected + def saml_config + Gitlab::Auth::Saml::Config + end + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end def external_users_enabled? - !Gitlab::Auth::Saml::Config.external_groups.nil? + !saml_config.external_groups.nil? end def auth_hash=(auth_hash) diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb new file mode 100644 index 00000000000..1893cb001b2 --- /dev/null +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -0,0 +1,33 @@ +module Gitlab + module Auth + class UserAccessDeniedReason + def initialize(user) + @user = user + end + + def rejection_message + case rejection_type + when :internal + "This action cannot be performed by internal users" + when :terms_not_accepted + "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ + "Please access GitLab from a web browser to accept these terms." + else + "Your account has been blocked." + end + end + + private + + def rejection_type + if @user.internal? + :internal + elsif @user.required_terms_not_accepted? + :terms_not_accepted + else + :blocked + end + end + end + end +end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index cf02030c577..c7993665421 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,9 +1,5 @@ module Gitlab module Auth - # - # Exceptions - # - AuthenticationError = Class.new(StandardError) MissingTokenError = Class.new(AuthenticationError) TokenNotFoundError = Class.new(AuthenticationError) @@ -29,13 +25,15 @@ module Gitlab current_request.env['warden']&.authenticate if verified_request? end - def find_user_from_rss_token - return unless current_request.path.ends_with?('.atom') || current_request.format.atom? + def find_user_from_feed_token + return unless rss_request? || ics_request? - token = current_request.params[:rss_token].presence + # NOTE: feed_token was renamed from rss_token but both needs to be supported because + # users might have already added the feed to their RSS reader before the rename + token = current_request.params[:feed_token].presence || current_request.params[:rss_token].presence return unless token - User.find_by_rss_token(token) || raise(UnauthorizedError) + User.find_by_feed_token(token) || raise(UnauthorizedError) end def find_user_from_access_token @@ -61,6 +59,12 @@ module Gitlab private + def route_authentication_setting + return {} unless respond_to?(:route_setting) + + route_setting(:authentication) || {} + end + def access_token strong_memoize(:access_token) do find_oauth_access_token || find_personal_access_token @@ -102,6 +106,14 @@ module Gitlab def current_request @current_request ||= ensure_action_dispatch_request(request) end + + def rss_request? + current_request.path.ends_with?('.atom') || current_request.format.atom? + end + + def ics_request? + current_request.path.ends_with?('.ics') || current_request.format.ics? + end end end end diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb new file mode 100644 index 00000000000..5a4e5b2c471 --- /dev/null +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ArchiveLegacyTraces + def perform(start_id, stop_id) + # This background migration directly refers to ::Ci::Build model which is defined in application code. + # In general, migration code should be isolated as much as possible in order to be idempotent. + # However, `archive!` method is too complicated to be replicated by coping its subsequent code. + # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 + ::Ci::Build.finished.without_archived_trace + .where(id: start_id..stop_id).find_each do |build| + begin + build.trace.archive! + rescue => e + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb new file mode 100644 index 00000000000..d3f366f3480 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentRename < CleanupConcurrentSchemaChange + RESCHEDULE_DELAY = 10.minutes + + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_rename(table, old_column, new_column) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb new file mode 100644 index 00000000000..54f77f184d5 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for cleaning up concurrent schema changes. + class CleanupConcurrentSchemaChange + include Database::MigrationHelpers + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(table, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + self.class.name, + [table, old_column, new_column] + ) + else + cleanup_concurrent_schema_change(table, old_column, new_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb index de622f657b2..48411095dbb 100644 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -2,52 +2,12 @@ module Gitlab module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentTypeChange - include Database::MigrationHelpers - + # Background migration for cleaning up a concurrent column type changeb. + class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange RESCHEDULE_DELAY = 10.minutes - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(:issues, new_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - 'CleanupConcurrentTypeChange', - [table, old_column, new_column] - ) - else - cleanup_concurrent_column_type_change(table, old_column) - end - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_type_change(table, old_column) end end end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index fd5cbf76e47..a357538a885 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -96,7 +96,7 @@ module Gitlab commit_hash.merge( merge_request_diff_id: merge_request_diff.id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha) + sha: sha_attribute.serialize(sha) ) end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 00000000000..22b0ac71920 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 00000000000..d0816ae3ed5 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb new file mode 100644 index 00000000000..94c65459a67 --- /dev/null +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillStoreUpload::Upload + .where(store: nil) + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 8fe4f1a2289..242e3143e71 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -12,6 +12,7 @@ module Gitlab class Build < ActiveRecord::Base self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled def ensure_stage!(attempts: 2) find_stage || create_stage! diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb new file mode 100644 index 00000000000..f90f35a913d --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class MigrateStageIndex + def perform(start_id, stop_id) + migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql| + ActiveRecord::Base.connection.execute(sql) + end + end + + private + + def migrate_stage_index_sql(start_id, stop_id) + if Gitlab::Database.postgresql? + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, last_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq ASC) AS index + FROM freqs + ) + + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL + else + <<~SQL + UPDATE ci_stages + SET position = + (SELECT stage_idx FROM ci_builds + WHERE ci_builds.stage_id = ci_stages.id + GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) + WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} + AND ci_stages.position IS NULL + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_import_state.rb b/lib/gitlab/background_migration/populate_import_state.rb new file mode 100644 index 00000000000..695a2a713c5 --- /dev/null +++ b/lib/gitlab/background_migration/populate_import_state.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration creates all the records on the + # import state table for projects that are considered imports or forks + class PopulateImportState + def perform(start_id, end_id) + move_attributes_data_to_import_state(start_id, end_id) + rescue ActiveRecord::RecordNotUnique + retry + end + + def move_attributes_data_to_import_state(start_id, end_id) + Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO project_mirror_data (project_id, status, jid, last_error) + SELECT id, import_status, import_jid, import_error + FROM projects + WHERE projects.import_status != 'none' + AND projects.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT id + FROM project_mirror_data + WHERE project_id = projects.id + ) + SQL + + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects + SET import_status = 'none' + WHERE import_status != 'none' + AND id BETWEEN #{start_id} AND #{end_id} + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 914a9e48a2f..522c69a0bb1 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -54,7 +54,8 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless UntrackedFile.connection.table_exists?(table_name) + + unless ActiveRecord::Base.connection.data_source_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true diff --git a/lib/gitlab/background_migration/rollback_import_state_data.rb b/lib/gitlab/background_migration/rollback_import_state_data.rb new file mode 100644 index 00000000000..a7c986747d8 --- /dev/null +++ b/lib/gitlab/background_migration/rollback_import_state_data.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration migrates all the data of import_state + # back to the projects table for projects that are considered imports or forks + class RollbackImportStateData + def perform(start_id, end_id) + move_attributes_data_to_project(start_id, end_id) + end + + def move_attributes_data_to_project(start_id, end_id) + Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}") + + if Gitlab::Database.mysql? + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects, project_mirror_data + SET + projects.import_status = project_mirror_data.status, + projects.import_jid = project_mirror_data.jid, + projects.import_error = project_mirror_data.last_error + WHERE project_mirror_data.project_id = projects.id + AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} + SQL + else + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects + SET + import_status = project_mirror_data.status, + import_jid = project_mirror_data.jid, + import_error = project_mirror_data.last_error + FROM project_mirror_data + WHERE project_mirror_data.project_id = projects.id + AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb new file mode 100644 index 00000000000..e5e8837221e --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures services which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnServices + class Service < ActiveRecord::Base + self.table_name = 'services' + + include ::EachBatch + + def self.services_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + Service.services_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb new file mode 100644 index 00000000000..171c8ef21b7 --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures hooks which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnWebhooks + class WebHook < ActiveRecord::Base + self.table_name = 'web_hooks' + + include ::EachBatch + + def self.hooks_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + WebHook.hooks_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 884a3de8f62..4ca5a78e068 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - project.repository.create_hooks + Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else @@ -75,10 +75,11 @@ module Gitlab end def mv_repo(project) - FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git')) + storage_path = storage_path_for_shard(project.repository_storage) + FileUtils.mv(repo_path, project.repository.path_to_repo) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) end true @@ -88,6 +89,10 @@ module Gitlab false end + def storage_path_for_shard(shard) + Gitlab.config.repositories.storages[shard].legacy_disk_path + end + def find_or_create_groups return nil unless group_path.present? diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb new file mode 100644 index 00000000000..e4227af25d2 --- /dev/null +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -0,0 +1,8 @@ +# This is a base controller for doorkeeper. +# It adds the `can?` helper used in the views. +module Gitlab + class BaseDoorkeeperController < ActionController::Base + include Gitlab::Allowable + helper_method :can? + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index bffbcb86137..f3999e690fa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) + gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/build_access.rb b/lib/gitlab/build_access.rb new file mode 100644 index 00000000000..08a8f846ca5 --- /dev/null +++ b/lib/gitlab/build_access.rb @@ -0,0 +1,12 @@ +module Gitlab + class BuildAccess < UserAccess + attr_accessor :user, :project + + # This bypasses the `can?(:access_git)`-check we normally do in `UserAccess` + # for CI. That way if a user was able to trigger a pipeline, then the + # build is allowed to clone the project. + def can_access_git? + true + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index dba37892863..add048d671e 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -40,7 +40,7 @@ module Gitlab end def self.cache_key_for_project(project) - "projects/#{project.id}/pipeline_status" + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status" end def self.update_for_pipeline(pipeline) diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..671b8e7e1b1 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,6 @@ module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 51ba09aa129..f76a6fb5f17 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -5,7 +5,7 @@ module Gitlab push_code: 'You are not allowed to push code to this project.', delete_default_branch: 'The default branch of a project cannot be deleted.', force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', - non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', push_protected_branch: 'You are not allowed to push code to protected branches on this project.', diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index 43a52b493bb..22310e313ac 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -37,7 +37,7 @@ module Gitlab def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + project.lfs_enabled? && newrev && oldrev && project.any_lfs_file_locks? end end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index c9c3050cfc2..87af4a90572 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -7,18 +7,10 @@ module Gitlab # Created or deleted branch return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - GitalyClient.migrate(:force_push) do |is_enabled| - if is_enabled - !project - .repository - .gitaly_commit_client - .ancestor?(oldrev, newrev) - else - Gitlab::Git::RevList.new( - project.repository.raw, oldrev: oldrev, newrev: newrev - ).missed_ref.present? - end - end + !project + .repository + .gitaly_commit_client + .ancestor?(oldrev, newrev) end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index f7276a380dc..f0e5773ec3c 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -15,8 +15,7 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.lfs_storage_project - .lfs_objects + existing_count = @project.all_lfs_objects .where(oid: new_lfs_pointers.map(&:lfs_oid)) .count diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3263790a876..3a197078d08 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -9,20 +9,16 @@ module Gitlab super(project, user, protocol) end - def message(rejected: false) + def message <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: - #{remote_url_message(rejected)} + git remote set-url origin #{url_to_repo} MESSAGE end - def permanent_redirect? - RedirectRoute.permanent.exists?(path: redirected_path) - end - private attr_reader :redirected_path @@ -30,18 +26,6 @@ module Gitlab def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end - - def remote_url_message(rejected) - if rejected - "git remote set-url origin #{url_to_repo} and try again." - else - "git remote set-url origin #{url_to_repo}" - end - end - - def url - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end end end end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index b20d374288f..782f6c4c0af 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index eadc0948d2f..4aa5dc89f47 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index c317291f29d..f09ba42c074 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb new file mode 100644 index 00000000000..9d2a362b7d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Policy + class Variables < Policy::Specification + def initialize(expressions) + @expressions = Array(expressions) + end + + def satisfied_by?(pipeline, seed) + variables = seed.to_resource.scoped_variables_hash + + statements = @expressions.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement + .new(statement, variables) + end + + statements.any?(&:truthful?) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 411f67f8ce7..0b1ebe4e048 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -14,7 +14,7 @@ module Gitlab self.new(:script).tap do |step| step.script = job.options[:before_script].to_a + job.options[:script].to_a step.script = job.commands.split("\n") if step.script.empty? - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ON_SUCCESS end end @@ -25,7 +25,7 @@ module Gitlab self.new(:after_script).tap do |step| step.script = after_script - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ALWAYS step.allow_failure = true end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index f7ff7ea212e..66ac4a40616 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,7 +4,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - def initialize(config) + # EE would override this and utilize opts argument + def initialize(config, opts = {}) @config = Loader.new(config).load! @global = Entry::Global.new(@config) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0027e9ec8c5..09e8e52b60f 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,15 +25,31 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes + attributes :refs, :kubernetes, :variables validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes] + validates :config, allowed_keys: %i[refs kubernetes variables] + validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] + validates :variables, array_of_strings: true + end + + def variables_expressions_syntax + return unless variables.is_a?(Array) + + statements = variables.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement) + end + + statements.each do |statement| + unless statement.valid? + errors.add(:variables, "Invalid expression syntax") + end + end end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 551483d0aaa..73f36735e35 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,7 +6,7 @@ module Gitlab def initialize(cron, cron_timezone = 'UTC') @cron = cron - @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name + @cron_timezone = timezone_name(cron_timezone) end def next_time_from(time) @@ -24,6 +24,12 @@ module Gitlab private + def timezone_name(timezone) + ActiveSupport::TimeZone.find_tzinfo(timezone).name + rescue TZInfo::InvalidTimezoneIdentifier + timezone + end + # NOTE: # cron_timezone can only accept timezones listed in TZInfo::Timezone. # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 70732d26bbd..b5eb0cfa2f0 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -14,7 +14,8 @@ module Gitlab trigger_requests: Array(@command.trigger_request), user: @command.current_user, pipeline_schedule: @command.schedule, - protected: @command.protected_ref? + protected: @command.protected_ref?, + variables_attributes: Array(@command.variables_attributes) ) @pipeline.set_config_source diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index a1849b01c5d..a53c80d34f7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -7,7 +7,7 @@ module Gitlab # rubocop:disable Naming/FileName :origin_ref, :checkout_sha, :after_sha, :before_sha, :trigger_request, :schedule, :ignore_skip_ci, :save_incompleted, - :seeds_block + :seeds_block, :variables_attributes ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..f4c8d5342c1 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,11 +9,16 @@ module Gitlab ::Ci::Pipeline.transaction do pipeline.save! - @command.seeds_block&.call(pipeline) - - ::Ci::CreatePipelineStagesService - .new(project, current_user) - .execute(pipeline) + ## + # Create environments before the pipeline starts. + # + pipeline.builds.each do |build| + if build.has_environment? + project.environments.find_or_create_by( + name: build.expanded_environment_name + ) + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb new file mode 100644 index 00000000000..f34c11ca3c2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -0,0 +1,44 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Populate < Chain::Base + include Chain::Helpers + + PopulateError = Class.new(StandardError) + + def perform! + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Populate pipeline with all stages, and stages with builds. + # + pipeline.stage_seeds.each do |stage| + pipeline.stages << stage.to_resource + end + + if pipeline.stages.none? + return error('No stages / jobs for this pipeline.') + end + + if pipeline.invalid? + return error('Failed to build the pipeline!') + end + + raise Populate::PopulateError if pipeline.persisted? + end + + def break? + pipeline.errors.any? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index 075504bcce5..a3bd2a5a23a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -16,11 +16,7 @@ module Gitlab @pipeline.drop!(:config_error) end - return error(@pipeline.yaml_errors) - end - - unless @pipeline.has_stage_seeds? - return error('No stages / jobs for this pipeline.') + error(@pipeline.yaml_errors) end end diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb new file mode 100644 index 00000000000..f57df7c5637 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -0,0 +1,10 @@ +module Gitlab + module Ci + module Pipeline + module Expression + ExpressionError = Class.new(StandardError) + RuntimeError = Class.new(ExpressionError) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb new file mode 100644 index 00000000000..10957598f76 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Matches < Lexeme::Operator + PATTERN = /=~/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + text = @left.evaluate(variables) + regexp = @right.evaluate(variables) + + regexp.scan(text.to_s).any? + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb new file mode 100644 index 00000000000..9b239c29ea4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + require_dependency 're2' + + class Pattern < Lexeme::Value + PATTERN = %r{^/.+/[ismU]*$}.freeze + + def initialize(regexp) + @value = regexp + + unless Gitlab::UntrustedRegexp.valid?(@value) + raise Lexer::SyntaxError, 'Invalid regular expression!' + end + end + + def evaluate(variables = {}) + Gitlab::UntrustedRegexp.fabricate(@value) + rescue RegexpError + raise Expression::RuntimeError, 'Invalid regular expression!' + end + + def self.build(string) + new(string) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 48bde213d44..346c92dc51e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -4,7 +4,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) @value = value diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index b781c15fd67..37643c8ef53 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -11,7 +11,7 @@ module Gitlab end def evaluate(variables = {}) - HashWithIndifferentAccess.new(variables).fetch(@name, nil) + variables.with_indifferent_access.fetch(@name, nil) end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index e1c68b7c3c2..4cacb1e62c9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -5,15 +5,17 @@ module Gitlab class Lexer include ::Gitlab::Utils::StrongMemoize + SyntaxError = Class.new(Expression::ExpressionError) + LEXEMES = [ Expression::Lexeme::Variable, Expression::Lexeme::String, + Expression::Lexeme::Pattern, Expression::Lexeme::Null, - Expression::Lexeme::Equals + Expression::Lexeme::Equals, + Expression::Lexeme::Matches ].freeze - SyntaxError = Class.new(Statement::StatementError) - MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 4f0e101b730..b36f1e0f865 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -3,23 +3,21 @@ module Gitlab module Pipeline module Expression class Statement - StatementError = Class.new(StandardError) + StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ + %w[variable], %w[variable equals string], %w[variable equals variable], %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable] + %w[variable matches pattern] ].freeze - def initialize(statement, pipeline) + def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) - - @variables = pipeline.variables.map do |variable| - [variable.key, variable.value] - end + @variables = variables.with_indifferent_access end def parse_tree @@ -35,6 +33,18 @@ module Gitlab def evaluate parse_tree.evaluate(@variables.to_h) end + + def truthful? + evaluate.present? + rescue Expression::ExpressionError + false + end + + def valid? + parse_tree.is_a?(Lexeme::Base) + rescue Expression::ExpressionError + false + end end end end diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb new file mode 100644 index 00000000000..db0a1ea4dab --- /dev/null +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + # Class for preloading data associated with pipelines such as commit + # authors. + class Preloader + def self.preload!(pipelines) + ## + # This preloads all commits at once, because `Ci::Pipeline#commit` is + # using a lazy batch loading, what results in only one batched Gitaly + # call. + # + pipelines.each(&:commit) + + pipelines.each do |pipeline| + self.new(pipeline).tap do |preloader| + preloader.preload_commit_authors + preloader.preload_pipeline_warnings + preloader.preload_stages_warnings + end + end + end + + def initialize(pipeline) + @pipeline = pipeline + end + + def preload_commit_authors + # This also preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. + @pipeline.commit.try(:lazy_author) + end + + def preload_pipeline_warnings + # This preloads the number of warnings for every pipeline, ensuring + # that Ci::Pipeline#has_warnings? doesn't execute any additional + # queries. + @pipeline.number_of_warnings + end + + def preload_stages_warnings + # This preloads the number of warnings for every stage, ensuring + # that Ci::Stage#has_warnings? doesn't execute any additional + # queries. + @pipeline.stages.each { |stage| stage.number_of_warnings } + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb new file mode 100644 index 00000000000..db9706924bb --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Base + def attributes + raise NotImplementedError + end + + def included? + raise NotImplementedError + end + + def to_resource + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb new file mode 100644 index 00000000000..6980b0b7aff --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -0,0 +1,48 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Build < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :dig, to: :@attributes + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @only = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:only)) + @except = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:except)) + end + + def included? + strong_memoize(:inclusion) do + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + end + end + + def attributes + @attributes.merge( + pipeline: @pipeline, + project: @pipeline.project, + user: @pipeline.user, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + ) + end + + def to_resource + strong_memoize(:resource) do + ::Ci::Build.new(attributes) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb new file mode 100644 index 00000000000..2b58d9863a0 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -0,0 +1,48 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Stage < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :size, to: :seeds + delegate :dig, to: :seeds + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @builds = attributes.fetch(:builds).map do |attributes| + Seed::Build.new(@pipeline, attributes) + end + end + + def attributes + { name: @attributes.fetch(:name), + position: @attributes.fetch(:index), + pipeline: @pipeline, + project: @pipeline.project } + end + + def seeds + strong_memoize(:seeds) do + @builds.select(&:included?) + end + end + + def included? + seeds.any? + end + + def to_resource + strong_memoize(:stage) do + ::Ci::Stage.new(attributes).tap do |stage| + seeds.each { |seed| stage.builds << seed.to_resource } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb deleted file mode 100644 index f33c87f554d..00000000000 --- a/lib/gitlab/ci/stage/seed.rb +++ /dev/null @@ -1,62 +0,0 @@ -module Gitlab - module Ci - module Stage - class Seed - include ::Gitlab::Utils::StrongMemoize - - attr_reader :pipeline - - delegate :project, to: :pipeline - delegate :size, to: :@jobs - - def initialize(pipeline, stage, jobs) - @pipeline = pipeline - @stage = { name: stage } - @jobs = jobs.to_a.dup - end - - def user=(current_user) - @jobs.map! do |attributes| - attributes.merge(user: current_user) - end - end - - def stage - @stage.merge(project: project) - end - - def builds - trigger = pipeline.trigger_requests.first - - @jobs.map do |attributes| - attributes.merge(project: project, - ref: pipeline.ref, - tag: pipeline.tag, - trigger_request: trigger, - protected: protected_ref?) - end - end - - def create! - pipeline.stages.create!(stage).tap do |stage| - builds_attributes = builds.map do |attributes| - attributes.merge(stage_id: stage.id) - end - - pipeline.builds.create!(builds_attributes).each do |build| - yield build if block_given? - end - end - end - - private - - def protected_ref? - strong_memoize(:protected_ref) do - project.protected_for?(pipeline.ref) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 2d9166d6bdd..024047d4983 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -23,6 +23,10 @@ module Gitlab 'Cancel' end + def action_button_title + _('Cancel this job') + end + def self.matches?(build, user) build.cancelable? end diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb new file mode 100644 index 00000000000..c83e2734a73 --- /dev/null +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Canceled < Status::Extended + def illustration + { + image: 'illustrations/canceled-job_empty.svg', + size: 'svg-430', + title: _('This job has been canceled') + } + end + + def self.matches?(build, user) + build.canceled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c0c7c7f5b5d..c1fc70ac266 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -3,6 +3,14 @@ module Gitlab module Status module Build module Common + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job does not have a trace.') + } + end + def has_details? can?(user, :read_build, subject) end diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb new file mode 100644 index 00000000000..5be8e9de425 --- /dev/null +++ b/lib/gitlab/ci/status/build/created.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Created < Status::Extended + def illustration + { + image: 'illustrations/job_not_triggered.svg', + size: 'svg-306', + title: _('This job has not been triggered yet'), + content: _('This job depends on upstream jobs that need to succeed in order for this job to be triggered') + } + end + + def self.matches?(build, user) + build.created? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb new file mode 100644 index 00000000000..495227c2ffb --- /dev/null +++ b/lib/gitlab/ci/status/build/erased.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Erased < Status::Extended + def illustration + { + image: 'illustrations/erased-log_empty.svg', + size: 'svg-430', + title: _('Job has been erased') + } + end + + def self.matches?(build, user) + build.erased? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index c852d607373..2b26ebb45a1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,12 +4,20 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [[Status::Build::Cancelable, + [[Status::Build::Erased, + Status::Build::Manual, + Status::Build::Canceled, + Status::Build::Created, + Status::Build::Pending, + Status::Build::Skipped], + [Status::Build::Cancelable, Status::Build::Retryable], + [Status::Build::Failed], [Status::Build::FailedAllowed, Status::Build::Play, Status::Build::Stop], - [Status::Build::Action]] + [Status::Build::Action], + [Status::Build::Retried]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb new file mode 100644 index 00000000000..155f4fc1343 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Status + module Build + class Failed < Status::Extended + REASONS = { + 'unknown_failure' => 'unknown failure', + 'script_failure' => 'script failure', + 'api_failure' => 'API failure', + 'stuck_or_timeout_failure' => 'stuck or timeout failure', + 'runner_system_failure' => 'runner system failure', + 'missing_dependency_failure' => 'missing dependency failure' + }.freeze + + def status_tooltip + base_message + end + + def badge_tooltip + base_message + end + + def self.matches?(build, user) + build.failed? + end + + private + + def base_message + "#{s_('CiStatusLabel|failed')} #{description}" + end + + def description + "<br> (#{REASONS[subject.failure_reason]})" + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index dc90f398c7e..ca0046fb1f7 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -4,7 +4,7 @@ module Gitlab module Build class FailedAllowed < Status::Extended def label - 'failed (allowed to fail)' + "failed #{allowed_to_fail_title}" end def icon @@ -15,9 +15,19 @@ module Gitlab 'failed_with_warnings' end + def status_tooltip + "#{@status.status_tooltip} #{allowed_to_fail_title}" + end + def self.matches?(build, user) build.failed? && build.allow_failure? end + + private + + def allowed_to_fail_title + "(allowed to fail)" + end end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb new file mode 100644 index 00000000000..042da6392d3 --- /dev/null +++ b/lib/gitlab/ci/status/build/manual.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Manual < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: _('This job requires a manual action'), + content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + } + end + + def self.matches?(build, user) + build.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb new file mode 100644 index 00000000000..9dd9a27ad57 --- /dev/null +++ b/lib/gitlab/ci/status/build/pending.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Pending < Status::Extended + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job has not started yet'), + content: _('This job is in pending state and is waiting to be picked by a runner') + } + end + + def self.matches?(build, user) + build.pending? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index b7b45466d3b..a8b9ebf0803 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -19,6 +19,10 @@ module Gitlab 'Play' end + def action_button_title + _('Trigger this manual action') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb new file mode 100644 index 00000000000..6e190e4ee3c --- /dev/null +++ b/lib/gitlab/ci/status/build/retried.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Build + class Retried < Status::Extended + def status_tooltip + @status.status_tooltip + " (retried)" + end + + def self.matches?(build, user) + build.retried? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 44ffe783e50..5aeb8e51480 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -15,6 +15,10 @@ module Gitlab 'Retry' end + def action_button_title + _('Retry this job') + end + def action_path retry_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb new file mode 100644 index 00000000000..3e678d0baee --- /dev/null +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Skipped < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job has been skipped') + } + end + + def self.matches?(build, user) + build.skipped? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 46e730797e4..dea838bfa39 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -19,6 +19,10 @@ module Gitlab 'Stop' end + def action_button_title + _('Stop this environment') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index d4fd83b93f8..9d6a2f51c11 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -22,6 +22,10 @@ module Gitlab raise NotImplementedError end + def illustration + raise NotImplementedError + end + def label raise NotImplementedError end @@ -57,6 +61,20 @@ module Gitlab def action_title raise NotImplementedError end + + def action_button_title + raise NotImplementedError + end + + # Hint that appears on all the pipeline graph tooltips and builds on the right sidebar in Job detail view + def status_tooltip + label + end + + # Hint that appears on the build badges + def badge_tooltip + subject.status + end end end end diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index bc99d925347..f60a7662075 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -8,7 +8,9 @@ module Gitlab end def details_path - project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) + project_pipeline_path(subject.pipeline.project, + subject.pipeline, + anchor: subject.name) end def has_action? diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index cedf4171ab1..a52d71225bb 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,6 +1,10 @@ module Gitlab module Ci class Trace + include ExclusiveLeaseGuard + + LEASE_TIMEOUT = 1.hour + ArchiveError = Class.new(StandardError) attr_reader :job @@ -36,16 +40,16 @@ module Gitlab end def set(data) - write do |stream| + write('w+b') do |stream| data = job.hide_secrets(data) stream.set(data) end end def append(data, offset) - write do |stream| + write('a+b') do |stream| current_length = stream.size - return -current_length unless current_length == offset + break current_length unless current_length == offset data = job.hide_secrets(data) stream.append(data, offset) @@ -54,13 +58,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || current_path.present? || old_trace.present? + trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do if trace_artifact trace_artifact.open + elsif job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") elsif old_trace @@ -73,9 +79,15 @@ module Gitlab stream&.close end - def write + def write(mode) stream = Gitlab::Ci::Trace::Stream.new do - File.open(ensure_path, "a+b") + if current_path + File.open(current_path, mode) + elsif Feature.enabled?('ci_enable_live_trace') + Gitlab::Ci::Trace::ChunkedIO.new(job) + else + File.open(ensure_path, mode) + end end yield(stream).tap do @@ -92,14 +104,28 @@ module Gitlab FileUtils.rm(trace_path, force: true) end + job.trace_chunks.fast_destroy_all job.erase_old_trace! end def archive! + try_obtain_lease do + unsafe_archive! + end + end + + private + + def unsafe_archive! raise ArchiveError, 'Already archived' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? - if current_path + if job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| + archive_stream!(stream) + stream.destroy! + end + elsif current_path File.open(current_path) do |stream| archive_stream!(stream) FileUtils.rm(current_path) @@ -112,11 +138,9 @@ module Gitlab end end - private - def archive_stream!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - create_job_trace!(job, clone_path) + create_build_trace!(job, clone_path) end end @@ -132,7 +156,7 @@ module Gitlab end end - def create_job_trace!(job, path) + def create_build_trace!(job, path) File.open(path) do |stream| job.create_job_artifacts_trace!( project: job.project, @@ -192,6 +216,16 @@ module Gitlab def trace_artifact job.job_artifacts_trace end + + # For ExclusiveLeaseGuard concern + def lease_key + @lease_key ||= "trace:archive:#{job.id}" + end + + # For ExclusiveLeaseGuard concern + def lease_timeout + LEASE_TIMEOUT + end end end end diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb new file mode 100644 index 00000000000..bfe0c2a2c26 --- /dev/null +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -0,0 +1,231 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class ChunkedIO + CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE + + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :build + attr_reader :tell, :size + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(build, &block) + @build = build + @chunks_cache = [] + @tell = 0 + @size = calculate_size + yield self if block_given? + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise ArgumentError, 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = "") + out = "" + + length ||= size - tell + + until length <= 0 || eof? + data = chunk_slice_from_offset + break if data.empty? + + chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize + end + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end + + out + end + + def readline + out = "" + + until eof? + data = chunk_slice_from_offset + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + start_pos = tell + + while tell < start_pos + data.bytesize + # get slice from current offset till the end where it falls into chunk + chunk_bytes = CHUNK_SIZE - chunk_offset + chunk_data = data.byteslice(tell - start_pos, chunk_bytes) + + # append data to chunk, overwriting from that point + ensure_chunk.append(chunk_data, chunk_offset) + + # move offsets within buffer + @tell += chunk_data.bytesize + @size = [size, tell].max + end + + tell - start_pos + ensure + invalidate_chunk_cache + end + + def truncate(offset) + raise ArgumentError, 'Outside of file' if offset > size || offset < 0 + return if offset == size # Skip the following process as it doesn't affect anything + + @tell = offset + @size = offset + + # remove all next chunks + trace_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all + + # truncate current chunk + current_chunk.truncate(chunk_offset) + ensure + invalidate_chunk_cache + end + + def flush + # no-op + end + + def present? + true + end + + def destroy! + trace_chunks.fast_destroy_all + @tell = @size = 0 + ensure + invalidate_chunk_cache + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def chunk_slice_from_offset + unless in_range? + current_chunk.tap do |chunk| + raise FailedToGetChunkError unless chunk + + @chunk = chunk.data + @chunk_range = chunk.range + end + end + + @chunk[chunk_offset..CHUNK_SIZE] + end + + def chunk_offset + tell % CHUNK_SIZE + end + + def chunk_index + tell / CHUNK_SIZE + end + + def chunk_start + chunk_index * CHUNK_SIZE + end + + def chunk_end + [chunk_start + CHUNK_SIZE, size].min + end + + def invalidate_chunk_cache + @chunks_cache = [] + end + + def current_chunk + @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) + end + + def build_chunk + @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + end + + def ensure_chunk + current_chunk || build_chunk + end + + def trace_chunks + ::Ci::BuildTraceChunk.where(build: build) + end + + def calculate_size + trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb new file mode 100644 index 00000000000..8788af57a67 --- /dev/null +++ b/lib/gitlab/ci/trace/http_io.rb @@ -0,0 +1,197 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = "") + out = "" + + length ||= size - tell + + until length <= 0 || eof? + data = get_chunk + break if data.empty? + + chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize + end + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index d52194f688b..a71040e5e56 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,9 +8,11 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true - delegate :valid?, to: :stream, as: :present?, allow_nil: true + delegate :valid?, to: :stream, allow_nil: true + + alias_method :present?, :valid? def initialize @stream = yield @@ -25,6 +27,10 @@ module Gitlab self.path.present? end + def path + self.stream.path if self.stream.respond_to?(:path) + end + def limit(last_bytes = LIMIT_SIZE) if last_bytes < size stream.seek(-last_bytes, IO::SEEK_END) @@ -33,6 +39,8 @@ module Gitlab end def append(data, offset) + data = data.force_encoding(Encoding::BINARY) + stream.truncate(offset) stream.seek(0, IO::SEEK_END) stream.write(data) @@ -40,8 +48,11 @@ module Gitlab end def set(data) - truncate(0) + data = data.force_encoding(Encoding::BINARY) + + stream.seek(0, IO::SEEK_SET) stream.write(data) + stream.truncate(data.bytesize) stream.flush() end @@ -81,7 +92,7 @@ module Gitlab match = matches.flatten.last coverage = match.gsub(/\d+(\.\d+)?/).first - return coverage if coverage.present? + return coverage if coverage.present? # rubocop:disable Cop/AvoidReturnFromBlocks end nil @@ -121,11 +132,11 @@ module Gitlab buf += debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| - yield(line.force_encoding('UTF-8')) + yield(line.force_encoding(Encoding.default_external)) end end - yield(debris.force_encoding('UTF-8')) unless debris.empty? + yield(debris.force_encoding(Encoding.default_external)) unless debris.empty? end def read_backward(length) diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 0deca55fe8f..ad30b3f427c 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -30,7 +30,13 @@ module Gitlab end def to_runner_variables - self.map(&:to_hash) + self.map(&:to_runner_variable) + end + + def to_hash + self.to_runner_variables + .map { |env| [env.fetch(:key), env.fetch(:value)] } + .to_h.with_indifferent_access end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 939912981e6..222aa06b800 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -3,12 +3,12 @@ module Gitlab module Variables class Collection class Item - def initialize(**options) + def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`value` must be of type String, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { - key: options.fetch(:key), - value: options.fetch(:value), - public: options.fetch(:public, true), - file: options.fetch(:files, false) + key: key, value: value, public: public, file: file } end @@ -17,7 +17,7 @@ module Gitlab end def ==(other) - to_hash == self.class.fabricate(other).to_hash + to_runner_variable == self.class.fabricate(other).to_runner_variable end ## @@ -25,7 +25,7 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - def to_hash + def to_runner_variable @variable.reject do |hash_key, hash_value| hash_key == :file && hash_value == false end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a7285ac8f9d..e829f2a95f8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :cache, :stages, :jobs - def initialize(config) - @ci_config = Gitlab::Ci::Config.new(config) + def initialize(config, opts = {}) + @ci_config = Gitlab::Ci::Config.new(config, opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -27,7 +27,7 @@ module Gitlab end def build_attributes(name) - job = @jobs[name.to_sym] || {} + job = @jobs.fetch(name.to_sym, {}) { stage_idx: @stages.index(job[:stage]), stage: job[:stage], @@ -53,37 +53,31 @@ module Gitlab }.compact } end - def pipeline_stage_builds(stage, pipeline) - selected_jobs = @jobs.select do |_, job| - next unless job[:stage] == stage - - only_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:only, {})) - except_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:except, {})) - - only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(pipeline) } - end - - selected_jobs.map { |_, job| build_attributes(job[:name]) } + def stage_builds_attributes(stage) + @jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } end - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) + def stages_attributes + @stages.uniq.map do |stage| + seeds = stage_builds_attributes(stage).map do |attributes| + job = @jobs.fetch(attributes[:name].to_sym) - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end + attributes + .merge(only: job.fetch(:only, {})) + .merge(except: job.fetch(:except, {})) + end - seeds.compact + { name: stage, index: @stages.index(stage), builds: seeds } + end end - def self.validation_message(content) + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? begin - Gitlab::Ci::YamlProcessor.new(content) + Gitlab::Ci::YamlProcessor.new(content, opts) nil rescue ValidationError => e e.message diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 3ccfd9a739d..65a65b67975 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -40,7 +40,10 @@ module Gitlab # when there are no conflict files. files.each(&:lines) files.any? - rescue Gitlab::Git::CommandError, Gitlab::Git::Conflict::Parser::UnresolvableError, Gitlab::Git::Conflict::Resolver::ConflictSideMissing + rescue Gitlab::Git::CommandError, + Gitlab::Git::Conflict::Parser::UnresolvableError, + Gitlab::Git::Conflict::Resolver::ConflictSideMissing, + Gitlab::Git::Conflict::File::UnsupportedEncoding false end cache_method :can_be_resolved_in_ui? diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index d7369060cc5..4c28489f45a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -85,7 +85,7 @@ module Gitlab .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") .where(conditions) - .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index e392a015b91..3cf35f499cd 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,8 +9,8 @@ module Gitlab end end - def fake_application_settings(defaults = ::ApplicationSetting.defaults) - Gitlab::FakeApplicationSettings.new(defaults) + def fake_application_settings(attributes = {}) + Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end def method_missing(name, *args, &block) @@ -24,44 +24,51 @@ module Gitlab private def ensure_application_settings! - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - cached_application_settings || uncached_application_settings end def cached_application_settings + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL - # In case Redis isn't running or the Redis UNIX socket file is not available + rescue + # In case Redis isn't running + # or the Redis UNIX socket file is not available + # or the DB is not running (we use migrations in the cache key) end end def uncached_application_settings return fake_application_settings unless connect_to_db? - db_settings = ::ApplicationSetting.current - + current_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks # and other callers from failing, use any loaded settings and return # defaults for missing columns. if ActiveRecord::Migrator.needs_migration? - defaults = ::ApplicationSetting.defaults - defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? - return fake_application_settings(defaults) + return fake_application_settings(current_settings&.attributes) end - return db_settings if db_settings.present? + return current_settings if current_settings.present? - ::ApplicationSetting.create_from_defaults || in_memory_application_settings + with_fallback_to_fake_application_settings do + ::ApplicationSetting.create_from_defaults || in_memory_application_settings + end end def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables - rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct + with_fallback_to_fake_application_settings do + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + + def with_fallback_to_fake_application_settings(&block) + yield + rescue + # In case the application_settings table is not created yet, or if a new + # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct fake_application_settings end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index bea78862757..550c1755a71 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -19,19 +19,11 @@ module Gitlab def count_commits return unless ref - repository = @project.repository.raw_repository - sha = @project.repository.commit(ref).sha - - cmd = %W(git --git-dir=#{repository.path} log) - cmd << '--format=%H' - cmd << "--after=#{@from.iso8601}" - cmd << sha - - output, status = Gitlab::Popen.popen(cmd) - - raise IOError, output unless status.zero? + gitaly_commit_client.commit_count(ref, after: @from) + end - output.lines.count + def gitaly_commit_client + Gitlab::GitalyClient::CommitService.new(@project.repository.raw_repository) end def ref diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 633de9f9776..bd14c7eece3 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -30,7 +30,7 @@ module Gitlab return unless enabled? @mutex.synchronize do - return thread if thread? + break thread if thread? @thread = Thread.new { start_working } end @@ -38,7 +38,7 @@ module Gitlab def stop @mutex.synchronize do - return unless thread? + break unless thread? stop_working diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 50fea1232af..f573368e572 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -9,6 +9,7 @@ module Gitlab # # data = { # object_kind: "note", + # event_type: "confidential_note", # user: { # name: String, # username: String, @@ -51,8 +52,11 @@ module Gitlab end def build_base_data(project, user, note) + event_type = note.confidential? ? 'confidential_note' : 'note' + base_data = { object_kind: "note", + event_type: event_type, user: user.hook_attrs, project_id: project.id, project: project.hook_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 76501dd50e8..4ad106e7b0a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -43,7 +43,7 @@ module Gitlab end def self.version - database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end def self.join_lateral_supported? @@ -188,8 +188,11 @@ module Gitlab end def self.cached_table_exists?(table_name) - # Rails 5 uses data_source_exists? instead of table_exists? - connection.schema_cache.table_exists?(table_name) + if Gitlab.rails5? + connection.schema_cache.data_source_exists?(table_name) + else + connection.schema_cache.table_exists?(table_name) + end end private_class_method :connection diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb new file mode 100644 index 00000000000..d7e3ce08b32 --- /dev/null +++ b/lib/gitlab/database/arel_methods.rb @@ -0,0 +1,18 @@ +module Gitlab + module Database + module ArelMethods + private + + # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer` + # was removed. + # Remove this file and inline this method when removing rails5? code. + def arel_update_manager + if Gitlab.rails5? + Arel::UpdateManager.new + else + Arel::UpdateManager.new(ActiveRecord::Base) + end + end + end + end +end diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb new file mode 100644 index 00000000000..5f549ed2b3c --- /dev/null +++ b/lib/gitlab/database/count.rb @@ -0,0 +1,86 @@ +# For large tables, PostgreSQL can take a long time to count rows due to MVCC. +# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +module Gitlab + module Database + module Count + CONNECTION_ERRORS = + if defined?(PG) + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid, + PG::Error + ].freeze + else + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid + ].freeze + end + + # Takes in an array of models and returns a Hash for the approximate + # counts for them. If the model's table has not been vacuumed or + # analyzed recently, simply run the Model.count to get the data. + # + # @param [Array] + # @return [Hash] of Model -> count mapping + def self.approximate_counts(models) + table_to_model_map = models.each_with_object({}) do |model, hash| + hash[model.table_name] = model + end + + table_names = table_to_model_map.keys + counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} + + # Convert table -> count to Model -> count + counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| + model = table_to_model_map[pair.first] + hash[model] = pair.second + end + + missing_tables = table_names - counts_by_table_name.keys + + missing_tables.each do |table| + model = table_to_model_map[table] + counts_by_model[model] = model.count + end + + counts_by_model + end + + # Returns a hash of the table names that have recently updated tuples. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.reltuples_from_recently_updated(table_names) + query = postgresql_estimate_query(table_names) + rows = [] + + # Querying tuple stats only works on the primary. Due to load + # balancing, we need to ensure this query hits the load balancer. The + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + rows = ActiveRecord::Base.connection.select_all(query) + end + + rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } + rescue *CONNECTION_ERRORS + {} + end + + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.postgresql_estimate_query(table_names) + time = "to_timestamp(#{1.hour.ago.to_i})" + <<~SQL + SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class + LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname + WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) + AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) + SQL + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 74fed447289..f64e3d53138 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -33,7 +33,13 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table + arel_from = if Gitlab.rails5? + arel_table.from + else + arel_table + end + + query = arel_from .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) .project(average([arel_table[column_sym]], 'median')) .where( @@ -143,8 +149,13 @@ module Gitlab .order(arel_table[column_sym]) ).as('row_id') - count = arel_table.from(arel_table.alias) - .project('COUNT(*)') + arel_from = if Gitlab.rails5? + arel_table.from.from(arel_table.alias) + else + arel_table.from(arel_table.alias) + end + + count = arel_from.project('COUNT(*)') .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) .as('ct') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 44ca434056f..4fe5b4cc835 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,6 +1,8 @@ module Gitlab module Database module MigrationHelpers + include Gitlab::Database::ArelMethods + BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time @@ -314,7 +316,7 @@ module Gitlab stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = Arel::UpdateManager.new(ActiveRecord::Base) + update_arel = arel_update_manager .table(table) .set([[table[column], value]]) .where(table[:id].gteq(start_id)) @@ -594,6 +596,97 @@ module Gitlab end end + # Renames a column using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `rename_column_concurrently` since it can complete its work in a much + # shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # rename_column_using_background_migration( + # :users, + # :feed_token, + # :rss_token + # ) + # + # table - The name of the database table containing the column. + # + # old - The old column name. + # + # new - The new column name. + # + # type - The type of the new column. If no type is given the old column's + # type is used. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def rename_column_using_background_migration( + table, + old_column, + new_column, + type: nil, + batch_size: 10_000, + interval: 10.minutes + ) + + check_trigger_permissions!(table) + + old_col = column_for(table, old_column) + new_type = type || old_col.type + max_index = 0 + + add_column(table, new_column, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new_column, old_col.default) if old_col.default + + install_rename_triggers(table, old_column, new_column) + + model = Class.new(ActiveRecord::Base) do + self.table_name = table + + include ::EachBatch + end + + # Schedule the jobs that will copy the data from the old column to the + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, old_column, new_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentRename', + [table, old_column, new_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') + end + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end @@ -860,7 +953,7 @@ into similar problems in the future (e.g. when new tables are created). # Each job is scheduled with a `delay_interval` in between. # If you use a small interval, then some jobs may run at the same time. # - # model_class - The table being iterated over + # model_class - The table or relation being iterated over # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) # batch_size - The maximum number of rows per job @@ -900,11 +993,42 @@ into similar problems in the future (e.g. when new tables are created). end end - # Rails' index_exists? doesn't work when you only give it a table and index - # name. As such we have to use some extra code to check if an index exists for - # a given name. + # Fetches indexes on a column by name for postgres. + # + # This will include indexes using an expression on the column, for example: + # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` + # + # For mysql, it falls back to the default ActiveRecord implementation that + # will not find custom indexes. But it will select by name without passing + # a column. + # + # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: + # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 + # + # Or this can be removed when we no longer support postgres < 9.5, so we + # can use `CREATE INDEX IF NOT EXISTS`. def index_exists_by_name?(table, index) - indexes(table).map(&:name).include?(index) + # We can't fall back to the normal `index_exists?` method because that + # does not find indexes without passing a column name. + if indexes(table).map(&:name).include?(index.to_s) + true + elsif Gitlab::Database.postgresql? + postgres_exists_by_name?(table, index) + else + false + end + end + + def postgres_exists_by_name?(table, name) + index_sql = <<~SQL + SELECT COUNT(*) + FROM pg_index + JOIN pg_class i ON (indexrelid=i.oid) + JOIN pg_class t ON (indrelid=t.oid) + WHERE i.relname = '#{name}' AND t.relname = '#{table}' + SQL + + connection.select_value(index_sql).to_i > 0 end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index fd4a8832ec2..26ae6966746 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -37,6 +37,7 @@ module Gitlab class Namespace < ActiveRecord::Base include MigrationClasses::Routable self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled belongs_to :parent, class_name: "#{MigrationClasses.name}::Namespace" has_one :route, as: :source @@ -74,7 +75,7 @@ module Gitlab }.freeze def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] + Gitlab.config.repositories.storages[repository_storage].legacy_disk_path end # Overridden to have the correct `source_type` for the `route` relation diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 1a697396ff1..14de28a1d08 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -3,6 +3,8 @@ module Gitlab module RenameReservedPathsMigration module V1 class RenameBase + include Gitlab::Database::ArelMethods + attr_reader :paths, :migration delegate :update_column_in_batches, @@ -62,10 +64,10 @@ module Gitlab old_full_path, new_full_path) - update = Arel::UpdateManager.new(ActiveRecord::Base) - .table(routes) - .set([[routes[:path], replace_statement]]) - .where(Arel::Nodes::SqlLiteral.new(filter)) + update = arel_update_manager + .table(routes) + .set([[routes[:path], replace_statement]]) + .where(Arel::Nodes::SqlLiteral.new(filter)) execute(update.to_sql) end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 05b86f32ce2..73971af6a74 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -62,21 +62,20 @@ module Gitlab end def move_repositories(namespace, old_full_path, new_full_path) - repo_paths_for_namespace(namespace).each do |repository_storage_path| + repo_shards_for_namespace(namespace).each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, old_full_path) + gitlab_shell.add_namespace(repository_storage, old_full_path) - unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) - message = "Exception moving path #{repository_storage_path} \ - from #{old_full_path} to #{new_full_path}" + unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) + message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" Rails.logger.error message end end end - def repo_paths_for_namespace(namespace) + def repo_shards_for_namespace(namespace) projects_for_namespace(namespace).distinct.select(:repository_storage) - .map(&:repository_storage_path) + .map(&:repository_storage) end def projects_for_namespace(namespace) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 979225dd216..827aeb12a02 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -51,7 +51,7 @@ module Gitlab end def move_repository(project, old_path, new_path) - unless gitlab_shell.mv_repository(project.repository_storage_path, + unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) Rails.logger.error "Error moving #{old_path} to #{new_path}" diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index d9400e04b83..b2d8ee81977 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,12 +1,20 @@ module Gitlab module Database - BINARY_TYPE = if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveRecord::Type::Binary - end + BINARY_TYPE = + if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` + # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b + # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. + if Gitlab.rails5? + ActiveModel::Type::Binary + else + ActiveRecord::Type::Binary + end + end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # @@ -16,18 +24,39 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # Casts binary data to a SHA1 in hexadecimal. + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. def type_cast_from_database(value) - value = super + unpack_sha(super) + end + + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. + def type_cast_for_database(value) + serialize(value) + end + # It is called from activerecord-5.0.6/lib/active_record/attribute.rb + # Remove this method when removing Gitlab.rails5? code.. + def deserialize(value) + value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) + + unpack_sha(value) + end + + # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. + # Casts binary data to a SHA1 in hexadecimal. + def unpack_sha(value) + # Uncomment this line when removing Gitlab.rails5? code. + # value = super value ? value.unpack(PACK_FORMAT)[0] : nil end # Casts a SHA1 in hexadecimal to the proper binary format. - def type_cast_for_database(value) + def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - super(arg) + Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 014854da55c..40bcfa20e7d 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -76,6 +76,16 @@ module Gitlab line_code(line) if line end + # Returns the raw diff content up to the given line index + def diff_hunk(diff_line) + diff_line_index = diff_line.index + # @@ (match) header is not kept if it's found in the top of the file, + # therefore we should keep an extra line on this scenario. + diff_line_index += 1 unless diff_lines.first.match? + + diff_lines.select { |line| line.index <= diff_line_index }.map(&:text).join("\n") + end + def old_sha diff_refs&.base_sha end @@ -120,11 +130,13 @@ module Gitlab # Array of Gitlab::Diff::Line objects def diff_lines - @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + @diff_lines ||= + Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a end def highlighted_diff_lines - @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight + @highlighted_diff_lines ||= + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted @@ -229,8 +241,33 @@ module Gitlab simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) end + # This adds the bottom match line to the array if needed. It contains + # the data to load more context lines. + def diff_lines_for_serializer + lines = highlighted_diff_lines + + return if lines.empty? + + last_line = lines.last + + if last_line.new_pos < total_blob_lines(blob) + match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) + lines.push(match_line) + end + + lines + end + private + def total_blob_lines(blob) + @total_lines ||= begin + line_count = blob.lines.size + line_count -= 1 if line_count > 0 && blob.lines.last.blank? + line_count + end + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index a6007ebf531..c79d8d3cb21 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -36,6 +36,8 @@ module Gitlab private def decorate_diff!(diff) + return diff if diff.is_a?(File) + Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 269016daac2..5c1baa19b66 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -33,10 +33,7 @@ module Gitlab # match the blob, which is a bug. But we shouldn't fail to render # completely in that case, even though we want to report the error. rescue RangeError => e - if Gitlab::Sentry.enabled? - Gitlab::Sentry.context - Raven.capture_exception(e) - end + Gitlab::Sentry.track_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/45441') end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 010b4be7b40..81e91ea0ab7 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,11 +1,14 @@ module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker + def initialize(line, rich_line = nil) + super(line, rich_line || line) + end + def mark(line_inline_diffs, mode: nil) - mark = super(line_inline_diffs) do |text, left:, right:| + super(line_inline_diffs) do |text, left:, right:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - mark.html_safe end private diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0603141e441..2b3ebfbb9ff 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,22 +1,26 @@ module Gitlab module Diff class Line - attr_reader :type, :index, :old_pos, :new_pos + attr_reader :line_code, :type, :index, :old_pos, :new_pos attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos, parent_file: nil) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos @parent_file = parent_file + + # When line code is not provided from cache store we build it + # using the parent_file(Diff::File or Conflict::File). + @line_code = line_code || calculate_line_code end def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos]) + new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code]) end def serialize_keys - @serialize_keys ||= %i(text type index old_pos new_pos) + @serialize_keys ||= %i(line_code text type index old_pos new_pos) end def to_hash @@ -53,25 +57,46 @@ module Gitlab %w[match new-nonewline old-nonewline].include?(type) end + def match? + type == :match + end + def discussable? !meta? end def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text + @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text @rich_text end + def meta_positions + return unless meta? + + { + old_pos: old_pos, + new_pos: new_pos + } + end + def as_json(opts = nil) { + line_code: line_code, type: type, old_line: old_line, new_line: new_line, text: text, - rich_text: rich_text || text + rich_text: rich_text || text, + meta_data: meta_positions } end + + private + + def calculate_line_code + @parent_file&.line_code(self) + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 8302f30a0a2..7ae7ed286ed 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -3,7 +3,7 @@ module Gitlab class Parser include Enumerable - def parse(lines) + def parse(lines, diff_file: nil) return [] if lines.blank? @lines = lines @@ -31,17 +31,17 @@ module Gitlab next if line_old <= 1 && line_new <= 1 # top of file - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 next elsif line[0] == '\\' type = "#{context}-nonewline" - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 else type = identification_type(line) - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 690b27cde81..978962ab2eb 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -12,6 +12,10 @@ module Gitlab :head_sha, :old_line, :new_line, + :width, + :height, + :x, + :y, :position_type, to: :formatter # A position can belong to a text line or to an image coordinate diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0fb71976883..8c72d00c1f3 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,66 +2,90 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze - EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze + CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze + IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb|locale/gitlab\.pot}i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== ============================================================ }.freeze + STAY_STRONG_LINK_TO_DOCS = %Q{ + Stay 💪! For more information, see + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + }.freeze THANKS_FOR_READING_BANNER = %Q{ ============================================================ ==================== THANKS FOR READING ==================== ============================================================\n }.freeze - attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found + attr_reader :ee_repo_dir, :patches_dir + attr_reader :ce_project_url, :ee_repo_url + attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found attr_reader :job_id, :failed_files - def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil) + def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @ce_project_url = ce_project_url - @ce_repo_url = "#{ce_project_url}.git" + @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee') @job_id = job_id end def check ensure_patches_dir - add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") - generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') + # We're generating the patch against the canonical-ce remote since forks' + # master branch are not necessarily up-to-date. + add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") - add_remote('canonical-ee', EE_REPO_URL) + ee_remotes.each do |key, url| + add_remote(key, url) + end + fetch(branch: 'master', depth: 20, remote: 'canonical-ee') status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! - step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) - generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') + step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee') ee_branch_compat_check! end delete_ee_branches_locally! - if status.nil? - true - else - false - end + status.nil? end end private + def fork? + ce_project_url != CANONICAL_CE_PROJECT_URL + end + + def ee_remotes + return @ee_remotes if defined?(@ee_remotes) + + remotes = + { + 'ee' => ee_repo_url, + 'canonical-ee' => CANONICAL_EE_REPO_URL + } + remotes.delete('ee') unless fork? + + @ee_remotes = remotes + end + def add_remote(name, url) step( "Adding the #{name} remote (#{url})", @@ -70,28 +94,32 @@ module Gitlab end def ensure_ee_repo - if Dir.exist?(ee_repo_dir) - step("#{ee_repo_dir} already exists") - else - step( - "Cloning #{EE_REPO_URL} into #{ee_repo_dir}", - %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}] - ) + unless clone_repo(ee_repo_url, ee_repo_dir) + # Fallback to using the canonical EE if there is no forked EE + clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir) end end + def clone_repo(url, dir) + _, status = step( + "Cloning #{url} into #{dir}", + %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}] + ) + status.zero? + end + def ensure_patches_dir FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch:, patch_path:, remote:) + def generate_patch(branch:, patch_path:, branch_remote:, master_remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch, master_remote: remote) + find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote) step( - "Generating the patch against #{remote}/master in #{patch_path}", - %W[git diff --binary #{remote}/master...origin/#{branch}] + "Generating the patch against #{master_remote}/master in #{patch_path}", + %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -109,23 +137,22 @@ module Gitlab end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) - - if status.zero? - @ee_branch_found = ee_branch_prefix - return + ee_remotes.keys.each do |remote| + [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch| + _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + + if status.zero? + @ee_remote_with_branch = remote + @ee_branch_found = branch + return true + end + end end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) - - if status.zero? - @ee_branch_found = ee_branch_suffix - else - puts - puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - throw(:halt_check, :ko) - end + throw(:halt_check, :ko) end def ee_branch_compat_check! @@ -181,10 +208,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found?(master_remote:, branch:) + def merge_base_found?(branch:, branch_remote:, master_remote:) step( "Finding merge base with #{master_remote}/master", - %W[git merge-base #{master_remote}/master origin/#{branch}] + %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -193,7 +220,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:, master_remote:) + def find_merge_base_with_master(branch:, branch_remote:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -202,10 +229,10 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: branch, depth: depth, remote: branch_remote) fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found?(master_remote: master_remote, branch: branch) + merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote) end raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success @@ -274,6 +301,13 @@ module Gitlab Gitlab::Popen.popen(cmd) end + # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains + # redacted credentials (e.g. "***:****") which are useless in instructions + # the job gives. + def ce_public_repo_url + "#{ce_project_url}.git" + end + def applies_cleanly_msg(branch) %Q{ #{PLEASE_READ_THIS_BANNER} @@ -288,13 +322,15 @@ module Gitlab end def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + ee_repos = ee_remotes.values.uniq + %Q{ #{PLEASE_READ_THIS_BANNER} 💥 Oh no! 💥 The `#{ce_branch}` branch does not apply cleanly to the current EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch - was found in the EE repository. + was found in #{ee_repos.join(' nor in ')}. If you're a community contributor, don't worry, someone from GitLab Inc. will take care of this, and you don't have to do anything. @@ -314,17 +350,17 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD - $ git fetch #{ce_repo_url} #{ce_branch} + $ git fetch #{ce_public_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick - You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. + Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. 2. Apply your branch's patch to EE # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ wget #{patch_url} && git apply --3way #{ce_patch_name} @@ -356,10 +392,9 @@ module Gitlab ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. - Once this is done, you can retry this failed build, and it should pass. + Once this is done, you can retry this failed job, and it should pass. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end @@ -371,16 +406,15 @@ module Gitlab The `#{ce_branch}` does not apply cleanly to the current EE/master, and even though a `#{ee_branch_found}` branch - exists in the EE repository, it does not apply cleanly either to + exists in #{ee_repo_url}, it does not apply cleanly either to EE/master! #{conflicting_files_msg} Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and - retry this build. + retry this job. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index a616a80e8f5..764f93f6d3d 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -14,7 +14,7 @@ module Gitlab end def can_handle? - !incoming_email_token.nil? + !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) end def execute @@ -47,7 +47,7 @@ module Gitlab project, author, title: mail.subject, - description: message + description: message_including_reply ).execute end end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 3436306e122..2f864f2082b 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -23,7 +23,8 @@ module Gitlab def execute raise ProjectNotFound unless project - validate_permission!(:create_merge_request) + validate_permission!(:create_merge_request_in) + validate_permission!(:create_merge_request_from) verify_record!( record: create_merge_request, diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 8eea33b9ab5..5791dbd0484 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -8,6 +8,7 @@ module Gitlab include ReplyProcessing delegate :project, to: :sent_notification, allow_nil: true + delegate :noteable, to: :sent_notification def can_handle? mail_key =~ /\A\w+\z/ @@ -18,7 +19,7 @@ module Gitlab validate_permission!(:create_note) - raise NoteableNotFoundError unless sent_notification.noteable + raise NoteableNotFoundError unless noteable raise EmptyEmailError if message.blank? verify_record!( diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 32c5caf93e8..38b1425364f 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -16,8 +16,12 @@ module Gitlab @message ||= process_message end - def process_message - message = ReplyParser.new(mail).execute.strip + def message_including_reply + @message_with_reply ||= process_message(trim_reply: false) + end + + def process_message(**kwargs) + message = ReplyParser.new(mail, **kwargs).execute.strip add_attachments(message) end @@ -32,8 +36,12 @@ module Gitlab def validate_permission!(permission) raise UserNotFoundError unless author raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) + + if project + raise ProjectNotFound unless author.can?(:read_project, project) + end + + raise UserNotAuthorizedError unless author.can?(permission, project || noteable) end def verify_record!(record:, invalid_exception:, record_name:) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 01c28d051ee..ae6b84607d6 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -4,8 +4,9 @@ module Gitlab class ReplyParser attr_accessor :message - def initialize(message) + def initialize(message, trim_reply: true) @message = message + @trim_reply = trim_reply end def execute @@ -13,7 +14,9 @@ module Gitlab encoding = body.encoding - body = EmailReplyTrimmer.trim(body) + if @trim_reply + body = EmailReplyTrimmer.trim(body) + end return '' unless body diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 6659efa0961..0b8f6cfe3cb 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -90,7 +90,7 @@ module Gitlab end def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "".encode("UTF-16BE")) .encode("UTF-8") .gsub("\0".encode("UTF-8"), "") end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 1d6f5bb5e1c..d5d35dbd97f 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -50,7 +50,7 @@ module Gitlab status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 - [status_code, { 'ETag' => etag }, []] + [status_code, { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' }, []] end def track_cache_miss(if_none_match, cached_value_present, route) diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb new file mode 100644 index 00000000000..d512fc58e46 --- /dev/null +++ b/lib/gitlab/favicon.rb @@ -0,0 +1,57 @@ +module Gitlab + class Favicon + class << self + def main + image_name = + if appearance_favicon.exists? + appearance_favicon.url + elsif Gitlab::Utils.to_boolean(ENV['CANARY']) + 'favicon-yellow.png' + elsif Rails.env.development? + 'favicon-blue.png' + else + 'favicon.png' + end + + ActionController::Base.helpers.image_path(image_name, host: host) + end + + def status_overlay(status_name) + path = File.join( + 'ci_favicons', + "#{status_name}.png" + ) + + ActionController::Base.helpers.image_path(path, host: host) + end + + def available_status_names + @available_status_names ||= begin + Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) + .map { |file| File.basename(file, '.png') } + .sort + end + end + + private + + # we only want to create full urls when there's a different asset_host + # configured. + def host + if Gitlab::Application.config.asset_host.nil? || Gitlab::Application.config.asset_host == Gitlab.config.gitlab.base_url + nil + else + Gitlab.config.gitlab.base_url + end + end + + def appearance + RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) + end + + def appearance_favicon + appearance.favicon + end + end + end +end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index cc2638172ec..49bc9c0b671 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -14,6 +14,7 @@ module Gitlab avatar: /\Alogo\.(png|jpg|gif)\z/, issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, + xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z}, # Configuration files gitignore: '.gitignore', diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 8c082c0c336..af8270c8db8 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -14,14 +14,21 @@ module Gitlab end def find(query) - by_content = find_by_content(query) + query = Gitlab::Search::Query.new(query) do + filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i } + end + + by_content = find_by_content(query.term) already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query, except: already_found) + by_filename = find_by_filename(query.term, except: already_found) - (by_content + by_filename) - .sort_by(&:filename) - .map { |blob| [blob.filename, blob] } + files = (by_content + by_filename) + .sort_by(&:filename) + + query.filter_results(files).map { |blob| [blob.filename, blob] } end private @@ -32,17 +39,13 @@ module Gitlab end def find_by_filename(query, except: []) - filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? - - blob_refs = filenames.map { |filename| [ref, filename] } - blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) + filenames = search_filenames(query, except) - blobs.map do |blob| + blobs(filenames).map do |blob| Gitlab::SearchResults::FoundBlob.new( id: blob.id, filename: blob.path, - basename: File.basename(blob.path), + basename: File.basename(blob.path, File.extname(blob.path)), ref: ref, startline: 1, data: blob.data, @@ -50,5 +53,21 @@ module Gitlab ) end end + + def search_filenames(query, except) + filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames + end + + def blob_refs(filenames) + filenames.map { |filename| [ref, filename] } + end + + def blobs(filenames) + Gitlab::Git::Blob.batch(repository, blob_refs(filenames), blob_size_limit: 1024) + end end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 1b74f735679..b6eeb5d9a2b 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -21,7 +21,7 @@ module Gitlab @text.gsub(@pattern) do |markdown| file = find_file(@source_project, $~[:secret], $~[:file]) - return markdown unless file.try(:exists?) + break markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) with_link_in_tmp_dir(file.file) do |open_tmp_file| diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d4e893b881c..55236a1122f 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,5 +1,11 @@ +require_dependency 'gitlab/encoding_helper' + module Gitlab module Git + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and + # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze @@ -56,7 +62,7 @@ module Gitlab end def version - Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) + Gitlab::Git::Version.git_version end def check_namespace!(*objects) diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb index d8aeabb6cba..08f4d7d4f5c 100644 --- a/lib/gitlab/git/attributes_parser.rb +++ b/lib/gitlab/git/attributes_parser.rb @@ -3,12 +3,8 @@ module Gitlab # Class for parsing Git attribute files and extracting the attributes for # file patterns. class AttributesParser - def initialize(attributes_data) + def initialize(attributes_data = "") @data = attributes_data || "" - - if @data.is_a?(File) - @patterns = parse_file - end end # Returns all the Git attributes for the given path. @@ -28,7 +24,7 @@ module Gitlab # Returns a Hash containing the file patterns and their attributes. def patterns - @patterns ||= parse_file + @patterns ||= parse_data end # Parses an attribute string. @@ -91,8 +87,8 @@ module Gitlab private - # Parses the Git attributes file. - def parse_file + # Parses the Git attributes file contents. + def parse_data pairs = [] comment = '#' diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 6d6ed065f79..e25e15f5c80 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -15,34 +15,16 @@ module Gitlab def each @blames.each do |blame| - yield( - Gitlab::Git::Commit.new(@repo, blame.commit), - blame.line - ) + yield(blame.commit, blame.line) end end private def load_blame - raw_output = @repo.gitaly_migrate(:blame) do |is_enabled| - if is_enabled - load_blame_by_gitaly - else - load_blame_by_shelling_out - end - end - - output = encode_utf8(raw_output) - process_raw_blame output - end - - def load_blame_by_gitaly - @repo.gitaly_commit_client.raw_blame(@sha, @path) - end + output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path)) - def load_blame_by_shelling_out - @repo.shell_blame(@sha, @path) + process_raw_blame(output) end def process_raw_blame(output) @@ -60,9 +42,8 @@ module Gitlab end end - # load all commits in single call - commits.keys.each do |key| - commits[key] = @repo.lookup(key) + Gitlab::Git::Commit.batch_by_oid(@repo, commits.keys).each do |commit| + commits[commit.sha] = commit end # get it together diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index eabcf46cf58..604bb11e712 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -21,13 +21,31 @@ module Gitlab attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self - def find(repository, sha, path) - Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| - if is_enabled - find_by_gitaly(repository, sha, path) - else - find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - end + def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') + path = '/' if path.empty? + name = File.basename(path) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) + return unless entry + + entry.data = "" if limit == 0 + + case entry.type + when :COMMIT + new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha) + when :BLOB + new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8), + path: path, commit_id: sha, binary: binary?(entry.data)) end end @@ -56,12 +74,18 @@ module Gitlab repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + find(repository, sha, path, limit: blob_size_limit) end end end end + # Returns an array of Blob instances just with the metadata, that means + # the data attribute has no content. + def batch_metadata(repository, blob_references) + batch(repository, blob_references, blob_size_limit: 0) + end + # Find LFS blobs given an array of sha ids # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set @@ -98,25 +122,22 @@ module Gitlab # file.rb # oid: 4a # # - # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' # - def find_entry_by_path(repository, root_id, path) + def find_entry_by_path(repository, root_id, *path_parts) root_tree = repository.lookup(root_id) - # Strip leading slashes - path[%r{^/*}] = '' - path_arr = path.split('/') entry = root_tree.find do |entry| - entry[:name] == path_arr[0] + entry[:name] == path_parts[0] end return nil unless entry - if path_arr.size > 1 + if path_parts.size > 1 return nil unless entry[:type] == :tree - path_arr.shift - find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + path_parts.shift + find_entry_by_path(repository, entry[:oid], *path_parts) else [:blob, :commit].include?(entry[:type]) ? entry : nil end @@ -133,82 +154,6 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - return unless path - - path = path.sub(%r{\A/*}, '') - path = '/' if path.empty? - name = File.basename(path) - - # Gitaly will think that setting the limit to 0 means unlimited, while - # the client might only need the metadata and thus set the limit to 0. - # In this method we'll then set the limit to 1, but clear the byte of data - # that we got back so for the outside world it looks like the limit was - # actually 0. - req_limit = limit == 0 ? 1 : limit - - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) - return unless entry - - entry.data = "" if limit == 0 - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path, limit:) - return unless path - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, path) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - def rugged_raw(repository, sha, limit:) blob = repository.lookup(sha) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 93037ed8d90..341768752dc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MAX_COMMIT_MESSAGE_DISPLAY_SIZE = 10.megabytes MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, @@ -59,13 +60,14 @@ module Gitlab # Some weird thing? return nil unless commit_id.is_a?(String) + # This saves us an RPC round trip. + return nil if commit_id.include?(':') + commit = repo.gitaly_migrate(:find_commit) do |is_enabled| if is_enabled repo.gitaly_commit_client.find_commit(commit_id) else - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil + rugged_find(repo, commit_id) end end @@ -76,6 +78,12 @@ module Gitlab nil end + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(Rugged::Commit) ? obj : nil + end + # Get last commit for HEAD # # Ex. @@ -231,7 +239,8 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.gitaly_migrate(:list_commits_by_oid) do |is_enabled| + repo.gitaly_migrate(:list_commits_by_oid, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled repo.gitaly_commit_client.list_commits_by_oid(oids) else @@ -296,11 +305,40 @@ module Gitlab nil end end + + def get_message(repository, commit_id) + BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + commit_ids = items.map { |i| i[:commit_id] } + + messages = get_messages(repository, commit_ids) + + messages.each do |commit_sha, message| + loader.call({ repository: repository, commit_id: commit_sha }, message) + end + end + end + end + + def get_messages(repository, commit_ids) + repository.gitaly_migrate(:commit_messages) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.get_commit_messages(commit_ids) + else + commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h + end + end + end end def initialize(repository, raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit + @repository = repository + @head = head + case raw_commit when Hash init_from_hash(raw_commit) @@ -311,9 +349,6 @@ module Gitlab else raise "Invalid raw commit type: #{raw_commit.class}" end - - @repository = repository - @head = head end def sha @@ -341,35 +376,16 @@ module Gitlab parent_ids.first end - # Shows the diff between the commit's parent and the commit. - # - # Cuts out the header and stats from #to_patch and returns only the diff. - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 - def to_diff - Gitlab::GitalyClient.migrate(:commit_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.patch(id) - else - rugged_diff_from_parent.patch - end - end - end - # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end + @repository.gitaly_commit_client.diff_from_parent(self, options) end + # Not to be called directly, but right now its used for tests and in old + # migrations def rugged_diff_from_parent(options = {}) options ||= {} break_rewrites = options[:break_rewrites] @@ -431,16 +447,6 @@ module Gitlab Gitlab::Git::CommitStats.new(@repository, self) end - def to_patch(options = {}) - begin - rugged_commit.to_mbox(options) - rescue Rugged::InvalidError => ex - if ex.message =~ /commit \w+ is a merge commit/i - 'Patch format is not currently supported for merge commits.' - end - end - end - # Get ref names collection # # Ex. @@ -485,6 +491,8 @@ module Gitlab end def tree_entry(path) + return unless path.present? + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| if is_migrated gitaly_tree_entry(path) @@ -540,7 +548,7 @@ module Gitlab # TODO: Once gitaly "takes over" Rugged consider separating the # subject from the message to make it clearer when there's one # available but not the other. - @message = (commit.body.presence || commit.subject).dup + @message = message_from_gitaly_body @authored_date = Time.at(commit.author.date.seconds).utc @author_name = commit.author.name.dup @author_email = commit.author.email.dup @@ -592,6 +600,25 @@ module Gitlab def refs(repo) repo.refs_hash[id] end + + def message_from_gitaly_body + return @raw_commit.subject.dup if @raw_commit.body_size.zero? + return @raw_commit.body.dup if full_body_fetched_from_gitaly? + + if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE + "#{@raw_commit.subject}\n\n--commit message is too big".strip + else + fetch_body_from_gitaly + end + end + + def full_body_fetched_from_gitaly? + @raw_commit.body.bytesize == @raw_commit.body_size + end + + def fetch_body_from_gitaly + self.class.get_message(@repository, id) + end end end end diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb new file mode 100644 index 00000000000..4198be7c9c9 --- /dev/null +++ b/lib/gitlab/git/committer_with_hooks.rb @@ -0,0 +1,47 @@ +module Gitlab + module Git + class CommitterWithHooks < Gollum::Committer + attr_reader :gl_wiki + + def initialize(gl_wiki, options = {}) + @gl_wiki = gl_wiki + super(gl_wiki.gollum_wiki, options) + end + + def commit + # TODO: Remove after 10.8 + return super unless allowed_to_run_hooks? + + result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( + @wiki.ref, + start_branch_name: @wiki.ref + ) do |start_commit| + super(false) + end + + result[:newrev] + rescue Gitlab::Git::PreReceiveError => e + message = "Custom Hook failed: #{e.message}" + raise Gitlab::Git::Wiki::OperationError, message + end + + private + + # TODO: Remove after 10.8 + def allowed_to_run_hooks? + @options[:user_id] != 0 && @options[:username].present? + end + + def git_user + @git_user ||= Gitlab::Git::User.new(@options[:username], + @options[:name], + @options[:email], + gitlab_id) + end + + def gitlab_id + Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) + end + end + end +end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index 2a9cf10a068..f08dab59ce4 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,17 +2,19 @@ module Gitlab module Git module Conflict class File + UnsupportedEncoding = Class.new(StandardError) + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid - attr_accessor :content + attr_accessor :raw_content - def initialize(repository, commit_oid, conflict, content) + def initialize(repository, commit_oid, conflict, raw_content) @repository = repository @commit_oid = commit_oid @their_path = conflict[:theirs][:path] @our_path = conflict[:ours][:path] @our_mode = conflict[:ours][:mode] - @content = content + @raw_content = raw_content end def lines @@ -29,6 +31,14 @@ module Gitlab end end + def content + @content ||= @raw_content.dup.force_encoding('UTF-8') + + raise UnsupportedEncoding unless @content.valid_encoding? + + @content + end + def type lines unless @type diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index 3effa9d2d31..fb5717dd556 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -4,7 +4,6 @@ module Gitlab class Parser UnresolvableError = Class.new(StandardError) UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) # Recoverable errors - the conflict can be resolved in an editor, but not with # sections. @@ -75,10 +74,6 @@ module Gitlab def validate_text!(text) raise UnmergeableFile if text.blank? # Typically a binary file raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? end def validate_delimiter!(condition) diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 07b7e811a34..c3cb0264112 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -23,7 +23,7 @@ module Gitlab end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) - rescue Rugged::OdbError, GRPC::BadStatus => e + rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a203587aec1..b58296375ef 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -249,7 +249,7 @@ module Gitlab if size >= SIZE_LIMIT too_large! - return true + return true # rubocop:disable Cop/AvoidReturnFromBlocks end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index a142ed6b2ef..5ff15a787f0 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -4,20 +4,14 @@ module Gitlab include Gitlab::Git::Popen include Gitlab::Utils::StrongMemoize - ShardNameNotFoundError = Class.new(StandardError) - - # Absolute path to directory where repositories are stored. - # Example: /home/git/repositories - attr_reader :shard_path + # Name of shard where repositories are stored. + # Example: nfs-file06 + attr_reader :shard_name # Relative path is a directory name for repository with .git at the end. # Example: gitlab-org/gitlab-test.git attr_reader :repository_relative_path - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - attr_reader :repository_absolute_path - # This is the path at which the gitlab-shell hooks directory can be found. # It's essential for integration between git and GitLab proper. All new # repositories should have their hooks directory symlinked here. @@ -25,13 +19,12 @@ module Gitlab attr_reader :logger - def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:) - @shard_path = shard_path + def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) + @shard_name = shard_name @repository_relative_path = repository_relative_path @logger = logger @global_hooks_path = global_hooks_path - @repository_absolute_path = File.join(shard_path, repository_relative_path) @output = StringIO.new end @@ -41,45 +34,42 @@ module Gitlab io.read end + # Absolute path to the repository. + # Example: /home/git/repositorities/gitlab-org/gitlab-test.git + # Probably will be removed when we fully migrate to Gitaly, part of + # https://gitlab.com/gitlab-org/gitaly/issues/1124. + def repository_absolute_path + strong_memoize(:repository_absolute_path) do + File.join(shard_path, repository_relative_path) + end + end + + def shard_path + strong_memoize(:shard_path) do + Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path + end + end + # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| - if is_enabled - gitaly_import_repository(source) - else - git_import_repository(source, timeout) - end - end + git_import_repository(source, timeout) end - def fork_repository(new_shard_path, new_repository_relative_path) - Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| - if is_enabled - gitaly_fork_repository(new_shard_path, new_repository_relative_path) - else - git_fork_repository(new_shard_path, new_repository_relative_path) - end - end + def fork_repository(new_shard_name, new_repository_relative_path) + git_fork_repository(new_shard_name, new_repository_relative_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option + cmd = fetch_remote_command(name, tags, prune, force) setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end end - - success end end @@ -205,17 +195,14 @@ module Gitlab private - def shard_name - strong_memoize(:shard_name) do - shard_name_from_shard_path(shard_path) + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') end end - def shard_name_from_shard_path(shard_path) - Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || - raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") - end - def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) @@ -242,18 +229,9 @@ module Gitlab true end - def gitaly_import_repository(source) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) - true - rescue GRPC::BadStatus => e - @output << e.message - false - end - - def git_fork_repository(new_shard_path, new_repository_relative_path) + def git_fork_repository(new_shard_name, new_repository_relative_path) from_path = repository_absolute_path + new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path to_path = File.join(new_shard_path, new_repository_relative_path) # The repository cannot already exist @@ -270,16 +248,6 @@ module Gitlab run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end - - def gitaly_fork_repository(new_shard_path, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) - rescue GRPC::BadStatus => e - logger.error "fork-repository failed: #{e.message}" - false - end end end end diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 4a43b9b444d..4b505312f60 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -46,6 +46,8 @@ module Gitlab iterator = State.new @content.split("\n").each_with_object(iterator) do |text, iterator| + text.chomp! + next if text =~ /^\s*#/ if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ @@ -55,7 +57,7 @@ module Gitlab next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ - value = $~[:value].chomp + value = $~[:value] iterator.set_attribute($~[:key], value) end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 24f027d8da4..94ff5b4980a 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -11,7 +11,7 @@ module Gitlab def initialize(name, repository) @name = name @repository = repository - @path = File.join(repo_path.strip, 'hooks', name) + @path = File.join(repo_path, 'hooks', name) end def repo_path @@ -95,13 +95,13 @@ module Gitlab args = [ref, oldrev, newrev] stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] + [status.success?, stderr.presence || stdout] end def retrieve_error_message(stderr, stdout) err_message = stderr.read err_message = err_message.blank? ? stdout.read : err_message - err_message.gsub(/\R/, "<br>").html_safe + err_message end end end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/hook_env.rb index 9d0b47a1a6d..455e8451c10 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -3,37 +3,39 @@ module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git - # commands may need. + # commands need during internal API calls made from Git push hooks. # # For example, in pre-receive hooks, new objects are put in a temporary # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved # (this would break push rules for instance). # # This class is thread-safe via RequestStore. - class Env + class HookEnv WHITELISTED_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze - def self.set(env) + def self.set(gl_repository, env) return unless RequestStore.active? - RequestStore.store[:gitlab_git_env] = whitelist_git_env(env) + raise "missing gl_repository" if gl_repository.blank? + + RequestStore.store[:gitlab_git_env] ||= {} + RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end - def self.all + def self.all(gl_repository) return {} unless RequestStore.active? - RequestStore.fetch(:gitlab_git_env) { {} } + h = RequestStore.fetch(:gitlab_git_env) { {} } + h.fetch(gl_repository, {}) end - def self.to_env_hash + def self.to_env_hash(gl_repository) env = {} - all.compact.each do |key, value| + all(gl_repository).compact.each do |key, value| value = value.join(File::PATH_SEPARATOR) if value.is_a?(Array) env[key.to_s] = value end @@ -41,10 +43,6 @@ module Gitlab env end - def self.[](key) - all[key] - end - def self.whitelist_git_env(env) env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index f302b852b35..e67cacdb95a 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -1,8 +1,6 @@ module Gitlab module Git class HooksService - PreReceiveError = Class.new(StandardError) - attr_accessor :oldrev, :newrev, :ref def execute(pusher, repository, oldrev, newrev, ref) diff --git a/lib/gitlab/git/info_attributes.rb b/lib/gitlab/git/info_attributes.rb deleted file mode 100644 index e79a440950b..00000000000 --- a/lib/gitlab/git/info_attributes.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Gitaly note: JV: not sure what to make of this class. Why does it use -# the full disk path of the repository to look up attributes This is -# problematic in Gitaly, because Gitaly hides the full disk path to the -# repository from gitlab-ce. - -module Gitlab - module Git - # Parses gitattributes at `$GIT_DIR/info/attributes` - # - # Unlike Rugged this parser only needs a single IO call (a call to `open`), - # vastly reducing the time spent in extracting attributes. - # - # This class _only_ supports parsing the attributes file located at - # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files - # (`.gitattributes` is copied to this particular path). - # - # Basic usage: - # - # attributes = Gitlab::Git::InfoAttributes.new(some_repo.path) - # - # attributes.attributes('README.md') # => { "eol" => "lf } - class InfoAttributes - delegate :attributes, :patterns, to: :parser - - # path - The path to the Git repository. - def initialize(path) - @repo_path = File.expand_path(path) - end - - def parser - @parser ||= begin - if File.exist?(attributes_path) - File.open(attributes_path, 'r') do |file_handle| - AttributesParser.new(file_handle) - end - else - AttributesParser.new("") - end - end - end - - private - - def attributes_path - @attributes_path ||= File.join(@repo_path, 'info/attributes') - end - end - end -end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index b9e5cf258f4..f0fab1e76a3 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,45 +7,11 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) - else - git_new_pointers(object_limit, not_in) - end - end + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) end def all_pointers - @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) - else - git_all_pointers - end - end - end - - private - - def git_new_pointers(object_limit, not_in) - @new_pointers ||= begin - rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| - object_ids = object_ids.take(object_limit) if object_limit - - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - end - - def git_all_pointers - rev_list.all_objects(require_path: true) do |object_ids| - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - - def rev_list - Gitlab::Git::RevList.new(@repository, newrev: @newrev) + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) end end end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 155cf52f050..57b82a37d6c 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[%r{^/*}] = '' + filename = filename.sub(%r{\A/*}, '') # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index c1767046ff0..f9f24ecc48d 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -25,7 +25,9 @@ module Gitlab stdin.close if lazy_block - return [lazy_block.call(stdout.lazy), 0] + cmd_output = lazy_block.call(stdout.lazy) + cmd_status = 0 + break else cmd_output << stdout.read end diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb new file mode 100644 index 00000000000..ac1ab7c39d5 --- /dev/null +++ b/lib/gitlab/git/pre_receive_error.rb @@ -0,0 +1,21 @@ +module Gitlab + module Git + # + # PreReceiveError is special because its message gets displayed to users + # in the web UI. To prevent XSS we sanitize the message on + # initialization. + class PreReceiveError < StandardError + def initialize(msg = '') + super(nlbr(msg)) + end + + private + + # In gitaly-ruby we override this method to do nothing, so that + # sanitization happens in gitlab-rails only. + def nlbr(str) + Gitlab::Utils.nlbr(str) + end + end + end +end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb new file mode 100644 index 00000000000..98de9328071 --- /dev/null +++ b/lib/gitlab/git/raw_diff_change.rb @@ -0,0 +1,71 @@ +module Gitlab + module Git + # This class behaves like a struct with fields :blob_id, :blob_size, :operation, :old_path, :new_path + # All those fields are (binary) strings or integers + class RawDiffChange + attr_reader :blob_id, :blob_size, :old_path, :new_path, :operation + + def initialize(raw_change) + if raw_change.is_a?(Gitaly::GetRawChangesResponse::RawChange) + @blob_id = raw_change.blob_id + @blob_size = raw_change.size + @old_path = raw_change.old_path.presence + @new_path = raw_change.new_path.presence + @operation = raw_change.operation&.downcase || :unknown + else + parse(raw_change) + end + end + + private + + # Input data has the following format: + # + # When a file has been modified: + # 7e3e39ebb9b2bf433b4ad17313770fbe4051649c 669 M\tfiles/ruby/popen.rb + # + # When a file has been renamed: + # 85bc2f9753afd5f4fc5d7c75f74f8d526f26b4f3 107 R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee + def parse(raw_change) + @blob_id, @blob_size, @raw_operation, raw_paths = raw_change.split(' ', 4) + @blob_size = @blob_size.to_i + @operation = extract_operation + @old_path, @new_path = extract_paths(raw_paths) + end + + def extract_paths(file_path) + case operation + when :copied, :renamed + file_path.split(/\t/) + when :deleted + [file_path, nil] + when :added + [nil, file_path] + else + [file_path, file_path] + end + end + + def extract_operation + return :unknown unless @raw_operation + + case @raw_operation[0] + when 'A' + :added + when 'C' + :copied + when 'D' + :deleted + when 'M' + :modified + when 'R' + :renamed + when 'T' + :type_changed + else + :unknown + end + end + end + end +end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index ebe46722890..e4743b4db0a 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -7,81 +7,8 @@ module Gitlab end def update(only_branches_matching: []) - @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| - if is_enabled - gitaly_update(only_branches_matching) - else - rugged_update(only_branches_matching) - end - end - end - - private - - def gitaly_update(only_branches_matching) - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end - - def rugged_update(only_branches_matching) - local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) - remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) - - updated_branches = changed_refs(local_branches, remote_branches) - push_branches(updated_branches.keys) if updated_branches.present? - - delete_refs(local_branches, remote_branches) - - local_tags = refs_obj(@repository.tags) - remote_tags = refs_obj(@repository.remote_tags(@ref_name)) - - updated_tags = changed_refs(local_tags, remote_tags) - @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? - - delete_refs(local_tags, remote_tags) - end - - def refs_obj(refs, only_refs_matching: []) - refs.each_with_object({}) do |ref, refs| - next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) - - refs[ref.name] = ref - end - end - - def changed_refs(local_refs, remote_refs) - local_refs.select do |ref_name, ref| - remote_ref = remote_refs[ref_name] - - remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target - end - end - - def push_branches(branches) - default_branch, branches = branches.partition do |branch| - @repository.root_ref == branch - end - - # Push the default branch first so it works fine when remote mirror is empty. - branches.unshift(*default_branch) - - @repository.push_remote_branches(@ref_name, branches) - end - - def delete_refs(local_refs, remote_refs) - refs = refs_to_delete(local_refs, remote_refs) - - @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? - end - - def refs_to_delete(local_refs, remote_refs) - default_branch_id = @repository.commit.id - - remote_refs.select do |remote_ref_name, remote_ref| - next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo - - remote_ref_id = remote_ref.dereferenced_target.try(:id) - - remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + @repository.wrapped_gitaly_errors do + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end end diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 6bd6e58feeb..f40e59a8dd0 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -12,7 +12,7 @@ module Gitlab # class. # class RemoteRepository - attr_reader :path, :relative_path, :gitaly_repository + attr_reader :relative_path, :gitaly_repository def initialize(repository) @relative_path = repository.relative_path @@ -21,7 +21,6 @@ module Gitlab # These instance variables will not be available in gitaly-ruby, where # we have no disk access to this repository. @repository = repository - @path = repository.path end def empty? @@ -69,6 +68,10 @@ module Gitlab env end + def path + @repository.path + end + private # Must return an object that responds to 'address' and 'storage'. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 208710b0935..88944cd62ea 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,8 @@ module Gitlab class Repository include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen + include Gitlab::EncodingHelper + include Gitlab::Utils::StrongMemoize ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ GIT_OBJECT_DIRECTORY @@ -18,18 +20,24 @@ module Gitlab GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze SEARCH_CONTEXT_LINES = 3 + # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 + # We copied these two prefixes into gitaly-go, so don't change these + # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX) REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze NoRepository = Class.new(StandardError) + InvalidRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the repository path @@ -72,9 +80,6 @@ module Gitlab end end - # Full path to repo - attr_reader :path - # Directory name of repo attr_reader :name @@ -93,31 +98,33 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - storage_path = Gitlab.config.repositories.storages[@storage]['path'] @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage_path, + storage, relative_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) - @path = File.join(storage_path, @relative_path) + @name = @relative_path.split("/").last - @attributes = Gitlab::Git::InfoAttributes.new(path) end def ==(other) - path == other.path + [storage, relative_path] == [other.storage, other.relative_path] + end + + def path + @path ||= File.join( + Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path + ) end # Default branch in the repository def root_ref - @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| - if is_enabled - gitaly_ref_client.default_branch_name - else - discover_default_branch - end - end + gitaly_ref_client.default_branch_name + rescue GRPC::NotFound => e + raise NoRepository.new(e.message) + rescue GRPC::Unknown => e + raise Gitlab::Git::CommandError.new(e.message) end def rugged @@ -137,37 +144,21 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_repository_client.exists? - else - circuit_breaker.perform do - File.exist?(File.join(@path, 'refs')) - end - end - end + gitaly_repository_client.exists? end # Returns an Array of branch names # sorted by name ASC def branch_names - gitaly_migrate(:branch_names) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names - else - branches.map(&:name) - end + wrapped_gitaly_errors do + gitaly_ref_client.branch_names end end # Returns an Array of Branches def branches - gitaly_migrate(:branches) do |is_enabled| - if is_enabled - gitaly_ref_client.branches - else - branches_filter - end + wrapped_gitaly_errors do + gitaly_ref_client.branches end end @@ -199,12 +190,8 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches) do |is_enabled| - if is_enabled - gitaly_ref_client.local_branches(sort_by: sort_by) - else - branches_filter(filter: :local, sort_by: sort_by) - end + wrapped_gitaly_errors do + gitaly_ref_client.local_branches(sort_by: sort_by) end end @@ -227,13 +214,13 @@ module Gitlab end end + def expire_has_local_branches_cache + clear_memoization(:has_local_branches) + end + def has_local_branches? - gitaly_migrate(:has_local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.has_local_branches? - else - has_local_branches_rugged? - end + strong_memoize(:has_local_branches) do + uncached_has_local_branches? end end @@ -244,18 +231,6 @@ module Gitlab # This refs by default not visible in project page and not cloned to client side. alias_method :has_visible_content?, :has_local_branches? - def has_local_branches_rugged? - rugged.branches.each(:local).any? do |ref| - begin - ref.name && ref.target # ensures the branch is valid - - true - rescue Rugged::ReferenceError - false - end - end - end - # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -269,12 +244,8 @@ module Gitlab # Returns an Array of tag names def tag_names - gitaly_migrate(:tag_names) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names - else - rugged.tags.map { |t| t.name } - end + wrapped_gitaly_errors do + gitaly_ref_client.tag_names end end @@ -282,12 +253,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - gitaly_migrate(:tags) do |is_enabled| - if is_enabled - tags_from_gitaly - else - tags_from_rugged - end + wrapped_gitaly_errors do + gitaly_ref_client.tags end end @@ -295,7 +262,8 @@ module Gitlab # # Ref names must start with `refs/`. def ref_exists?(ref_name) - gitaly_migrate(:ref_exists) do |is_enabled| + gitaly_migrate(:ref_exists, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_exists?(ref_name) else @@ -308,7 +276,7 @@ module Gitlab # # name - The name of the tag as a String. def tag_exists?(name) - gitaly_migrate(:ref_exists_tags) do |is_enabled| + gitaly_migrate(:ref_exists_tags, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_exists?("refs/tags/#{name}") else @@ -321,7 +289,7 @@ module Gitlab # # name - The name of the branch as a String. def branch_exists?(name) - gitaly_migrate(:ref_exists_branches) do |is_enabled| + gitaly_migrate(:ref_exists_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_exists?("refs/heads/#{name}") else @@ -362,58 +330,56 @@ module Gitlab end.map(&:name) end - # Discovers the default branch based on the repository's available branches - # - # - If no branches are present, returns nil - # - If one branch is present, returns its name - # - If two or more branches are present, returns current HEAD or master or first branch - def discover_default_branch - names = branch_names - - return if names.empty? - - return names[0] if names.length == 1 - - if rugged_head - extracted_name = Ref.extract_branch_name(rugged_head.name) - - return extracted_name if names.include?(extracted_name) - end - - if names.include?('master') - 'master' - else - names[0] - end - end - def rugged_head rugged.head rescue Rugged::ReferenceError nil end - def archive_prefix(ref, sha) - project_name = self.name.chomp('.git') - "#{project_name}-#{ref.tr('/', '-')}-#{sha}" - end - - def archive_metadata(ref, storage_path, format = "tar.gz") + def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id) + prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) { - 'RepoPath' => path, 'ArchivePrefix' => prefix, - 'ArchivePath' => archive_file_path(prefix, storage_path, format), - 'CommitId' => commit.id + 'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format), + 'CommitId' => commit.id, + 'GitalyRepository' => gitaly_repository.to_h } end - def archive_file_path(name, storage_path, format = "tar.gz") + # This is both the filename of the archive (missing the extension) and the + # name of the top-level member of the archive under which all files go + def archive_prefix(ref, sha, project_path, append_sha:) + append_sha = (ref != sha) if append_sha.nil? + + formatted_ref = ref.tr('/', '-') + + prefix_segments = [project_path, formatted_ref] + prefix_segments << sha if append_sha + + prefix_segments.join('-') + end + private :archive_prefix + + # The full path on disk where the archive should be stored. This is used + # to cache the archive between requests. + # + # The path is a global namespace, so needs to be globally unique. This is + # achieved by including `gl_repository` in the path. + # + # Archives relating to a particular ref when the SHA is not present in the + # filename must be invalidated when the ref is updated to point to a new + # SHA. This is achieved by including the SHA in the path. + # + # As this is a full path on disk, it is not "cloud native". This should + # be resolved by either removing the cache, or moving the implementation + # into Gitaly and removing the ArchivePath parameter from the git-archive + # senddata response. + def archive_file_path(storage_path, sha, name, format = "tar.gz") # Build file path return nil unless name @@ -431,18 +397,13 @@ module Gitlab end file_name = "#{name}.#{extension}" - File.join(storage_path, self.name, file_name) + File.join(storage_path, self.gl_repository, sha, file_name) end + private :archive_file_path # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end @@ -505,21 +466,25 @@ module Gitlab end def count_commits(options) - count_commits_options = process_count_commits_options(options) + options = process_count_commits_options(options.dup) - gitaly_migrate(:count_commits) do |is_enabled| - if is_enabled - count_commits_by_gitaly(count_commits_options) + wrapped_gitaly_errors do + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] else - count_commits_by_shelling_out(count_commits_options) + gitaly_commit_client.commit_count(options[:ref], options) end end end - def sha_from_ref(ref) - rev_parse_target(ref).oid - end - # Return the object that +revspec+ points to. If +revspec+ is an # annotated tag, then return the tag's target instead. def rev_parse_target(revspec) @@ -553,6 +518,26 @@ module Gitlab count_commits(from: from, to: to, **options) end + # old_rev and new_rev are commit ID's + # the result of this method is an array of Gitlab::Git::RawDiffChange + def raw_changes_between(old_rev, new_rev) + @raw_changes_between ||= {} + + @raw_changes_between[[old_rev, new_rev]] ||= + begin + return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA + + wrapped_gitaly_errors do + gitaly_repository_client.raw_changes_between(old_rev, new_rev) + .each_with_object([]) do |msg, arr| + msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) } + end + end + end + rescue ArgumentError => e + raise Gitlab::Git::Repository::GitError.new(e) + end + # Returns the SHA of the most recent common ancestor of +from+ and +to+ def merge_base(from, to) gitaly_migrate(:merge_base) do |is_enabled| @@ -564,24 +549,9 @@ module Gitlab end end - # Gitaly note: JV: check gitlab-ee before removing this method. - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| - if is_enabled - gitaly_commit_client.ancestor?(from, to) - else - rugged_is_ancestor?(from, to) - end - end + gitaly_commit_client.ancestor?(from, to) end def merged_branch_names(branch_names = []) @@ -622,17 +592,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end # Get refs hash which key is is the commit id @@ -678,15 +638,9 @@ module Gitlab end # Return total commits count accessible from passed ref - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - gitaly_migrate(:commit_count) do |is_enabled| - if is_enabled - gitaly_commit_client.commit_count(ref) - else - rugged_commit_count(ref) - end + wrapped_gitaly_errors do + gitaly_commit_client.commit_count(ref) end end @@ -715,13 +669,9 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_migrate(:operation_user_create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_add_branch(branch_name, user, target) - else - rugged_add_branch(branch_name, user, target) - end - end + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex end def add_tag(tag_name, user:, target:, message: nil) @@ -734,6 +684,10 @@ module Gitlab end end + def update_branch(branch_name, user:, newrev:, oldrev:) + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + def rm_branch(branch_name, user:) gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled @@ -888,7 +842,8 @@ module Gitlab end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs) do |is_enabled| + gitaly_migrate(:delete_refs, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_delete_refs(*ref_names) else @@ -964,13 +919,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -986,11 +935,18 @@ module Gitlab raise InvalidRef end + def info_attributes + return @info_attributes if @info_attributes + + content = gitaly_repository_client.info_attributes + @info_attributes = AttributesParser.new(content) + end + # Returns the Git attributes for the given file path. # # See `Gitlab::Git::Attributes` for more information. def attributes(path) - @attributes.attributes(path) + info_attributes.attributes(path) end def gitattribute(path, name) @@ -1011,44 +967,14 @@ module Gitlab end def languages(ref = nil) - gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.languages(ref) - else - ref ||= rugged.head.target_id - languages = Linguist::Repository.new(rugged, ref).languages - total = languages.map(&:last).sum - - languages = languages.map do |language| - name, share = language - color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" - { - value: (share.to_f * 100 / total).round(2), - label: name, - color: color, - highlight: color - } - end - - languages.sort do |x, y| - y[:value] <=> x[:value] - end - end + wrapped_gitaly_errors do + gitaly_commit_client.languages(ref) end end def license_short_name - gitaly_migrate(:license_short_name) do |is_enabled| - if is_enabled - gitaly_repository_client.license_short_name - else - begin - # The licensee gem creates a Rugged object from the path: - # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb - Licensee.license(path).try(:key) - rescue Rugged::Error - end - end + wrapped_gitaly_errors do + gitaly_repository_client.license_short_name end end @@ -1105,16 +1031,18 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - with_repo_branch_commit(source_repository, source_branch_name) do |commit| - break unless commit - - Gitlab::Git::Compare.new( - self, - target_branch_name, - commit.sha, - straight: straight - ) - end + tmp_ref = "refs/tmp/#{SecureRandom.hex}" + + return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) + + Gitlab::Git::Compare.new( + self, + target_branch_name, + tmp_ref, + straight: straight + ) + ensure + delete_refs(tmp_ref) end def write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1137,6 +1065,8 @@ module Gitlab if is_enabled gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) else + # When removing this code, also remove source_repository#path + # to remove deprecated method calls local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) end end @@ -1196,16 +1126,11 @@ module Gitlab end def create_from_bundle(bundle_path) - gitaly_migrate(:create_repo_from_bundle) do |is_enabled| - if is_enabled - gitaly_repository_client.create_from_bundle(bundle_path) - else - run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) - self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) - end - end + gitaly_repository_client.create_from_bundle(bundle_path) + end - true + def create_from_snapshot(url, auth) + gitaly_repository_client.create_from_snapshot(url, auth) end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) @@ -1227,12 +1152,8 @@ module Gitlab end def rebase_in_progress?(rebase_id) - gitaly_migrate(:rebase_in_progress) do |is_enabled| - if is_enabled - gitaly_repository_client.rebase_in_progress?(rebase_id) - else - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.rebase_in_progress?(rebase_id) end end @@ -1248,12 +1169,8 @@ module Gitlab end def squash_in_progress?(squash_id) - gitaly_migrate(:squash_in_progress) do |is_enabled| - if is_enabled - gitaly_repository_client.squash_in_progress?(squash_id) - else - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.squash_in_progress?(squash_id) end end @@ -1309,12 +1226,11 @@ module Gitlab def write_config(full_path:) return unless full_path.present? - gitaly_migrate(:write_config) do |is_enabled| - if is_enabled - gitaly_repository_client.write_config(full_path: full_path) - else - rugged_write_config(full_path: full_path) - end + # This guard avoids Gitaly log/error spam + raise NoRepository, 'repository does not exist' unless exists? + + wrapped_gitaly_errors do + gitaly_repository_client.write_config(full_path: full_path) end end @@ -1364,53 +1280,77 @@ module Gitlab raise CommandError.new(e) end - def branch_names_contains_sha(sha) - gitaly_migrate(:branch_names_contains_sha) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names_contains_sha(sha) - else - refs_contains_sha('refs/heads/', sha) - end + def wrapped_gitaly_errors(&block) + yield block + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + + def clean_stale_repository_files + gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_repository_client.cleanup if is_enabled && exists? end + rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup + Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") + Gitlab::Metrics.counter( + :failed_repository_cleanup_total, + 'Number of failed repository cleanup events' + ).increment + end + + def branch_names_contains_sha(sha) + gitaly_ref_client.branch_names_contains_sha(sha) end def tag_names_contains_sha(sha) - gitaly_migrate(:tag_names_contains_sha) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names_contains_sha(sha) - else - refs_contains_sha('refs/tags/', sha) - end - end + gitaly_ref_client.tag_names_contains_sha(sha) end def search_files_by_content(query, ref) return [] if empty? || query.blank? - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) - - run_git(args).first.scrub.split(/^--\n/) - end + safe_query = Regexp.escape(query) + ref ||= root_ref - def can_be_merged?(source_sha, target_branch) - gitaly_migrate(:can_be_merged) do |is_enabled| + gitaly_migrate(:search_files_by_content) do |is_enabled| if is_enabled - gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) + gitaly_repository_client.search_files_by_content(ref, safe_query) else - rugged_can_be_merged?(source_sha, target_branch) + offset = 2 + args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) + + run_git(args).first.scrub.split(/^--\n/) end end end + def can_be_merged?(source_sha, target_branch) + if target_sha = find_branch(target_branch, true)&.target + !gitaly_conflicts_client(source_sha, target_sha).conflicts? + else + false + end + end + def search_files_by_name(query, ref) safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + ref ||= root_ref return [] if empty? || safe_query.blank? - args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{safe_query}) + gitaly_migrate(:search_files_by_name) do |is_enabled| + if is_enabled + gitaly_repository_client.search_files_by_name(ref, safe_query) + else + args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - run_git(args).first.lines.map(&:strip) + run_git(args).first.lines.map(&:strip) + end + end end def find_commits_by_message(query, ref, path, limit, offset) @@ -1438,7 +1378,7 @@ module Gitlab end end - def rev_list(including: [], excluding: [], objects: false, &block) + def rev_list(including: [], excluding: [], options: [], objects: false, &block) args = ['rev-list'] args.push(*rev_list_param(including)) @@ -1451,42 +1391,63 @@ module Gitlab args.push('--objects') if objects + if options.any? + args.push(*options) + end + run_git!(args, lazy_block: block) end - def missed_ref(oldrev, newrev) - run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) - end + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) - private + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - def local_write_ref(ref_path, ref, old_ref: nil, shell: true) - if shell - shell_write_ref(ref_path, ref, old_ref) + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) else - rugged_write_ref(ref_path, ref) + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) end - end - def refs_contains_sha(refs_prefix, sha) - refs_prefix << "/" unless refs_prefix.ends_with?('/') + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end - # By forcing the output to %(refname) each line wiht a ref will start with - # the ref prefix. All other lines can be discarded. - args = %W(for-each-ref --contains=#{sha} --format=%(refname) #{refs_prefix}) - names, code = run_git(args) + def checksum + # The exists? RPC is much cheaper, so we perform this request first + raise NoRepository, "Repository does not exists" unless exists? - return [] unless code.zero? + gitaly_repository_client.calculate_checksum + rescue GRPC::NotFound + raise NoRepository # Guard against data races. + end - refs = [] - left_slice_count = refs_prefix.length - names.lines.each do |line| - next unless line.start_with?(refs_prefix) + private - refs << line.rstrip[left_slice_count..-1] + def uncached_has_local_branches? + wrapped_gitaly_errors do + gitaly_repository_client.has_local_branches? end + end - refs + def local_write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end end def rugged_write_config(full_path:) @@ -1540,44 +1501,6 @@ module Gitlab end end - def fresh_worktree?(path) - File.exist?(path) && !clean_stuck_worktree(path) - end - - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) - base_args = %w(worktree add --detach) - - # Note that we _don't_ want to test for `.present?` here: If the caller - # passes an non nil empty value it means it still wants sparse checkout - # but just isn't interested in any file, perhaps because it wants to - # checkout files in by a changeset but that changeset only adds files. - if sparse_checkout_files - # Create worktree without checking out - run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - - configure_sparse_checkout(worktree_git_path, sparse_checkout_files) - - # After sparse checkout configuration, checkout `branch` in worktree - run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) - else - # Create worktree and checkout `branch` in it - run_git!(base_args + [worktree_path, branch], env: env) - end - - yield - ensure - FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) - FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) - end - - def clean_stuck_worktree(path) - return false unless File.mtime(path) < 15.minutes.ago - - FileUtils.rm_rf(path) - true - end - # Adding a worktree means checking out the repository. For large repos, # this can be very expensive, so set up sparse checkout for the worktree # to only check out the files we're interested in. @@ -1620,20 +1543,6 @@ module Gitlab } end - # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. - def branches_filter(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) - end - def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} @@ -1748,21 +1657,11 @@ module Gitlab end def alternate_object_directories - relative_paths = relative_object_directories - - if relative_paths.any? - relative_paths.map { |d| File.join(path, d) } - else - absolute_object_directories.flat_map { |d| d.split(File::PATH_SEPARATOR) } - end + relative_object_directories.map { |d| File.join(path, d) } end def relative_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def absolute_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).flatten.compact + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact end # Get the content of a blob for a given commit. If the blob is a commit @@ -1855,132 +1754,11 @@ module Gitlab end end - def tags_from_rugged - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end - end - - target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) - end.sort_by(&:name) - end - def last_commit_for_path_by_rugged(sha, path) sha = last_commit_id_for_path_by_shelling_out(sha, path) commit(sha) end - def tags_from_gitaly - gitaly_ref_client.tags - end - - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def count_commits_by_gitaly(options) - if options[:left_right] - from = options[:from] - to = options[:to] - - right_count = gitaly_commit_client - .commit_count("#{from}..#{to}", options) - left_count = gitaly_commit_client - .commit_count("#{to}..#{from}", options) - - [left_count, right_count] - else - gitaly_commit_client.commit_count(options[:ref], options) - end - end - - def count_commits_by_shelling_out(options) - cmd = count_commits_shelling_command(options) - - raw_output, _status = run_git(cmd) - - process_count_commits_raw_output(raw_output, options) - end - - def count_commits_shelling_command(options) - cmd = %w[rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << "--max-count=#{options[:max_count]}" if options[:max_count] - cmd << "--left-right" if options[:left_right] - cmd << '--count' - - cmd << if options[:all] - '--all' - elsif options[:ref] - options[:ref] - else - raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" - end - - cmd += %W[-- #{options[:path]}] if options[:path].present? - cmd - end - - def process_count_commits_raw_output(raw_output, options) - if options[:left_right] - result = raw_output.scan(/\d+/).map(&:to_i) - - if result.sum != options[:max_count] - result - else # Reaching max count, right is not accurate - right_option = - process_count_commits_options(options - .except(:left_right, :from, :to) - .merge(ref: options[:to])) - - right = count_commits_by_shelling_out(right_option) - - [result.first, right] # left should be accurate in the first call - end - else - raw_output.to_i - end - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -2113,22 +1891,6 @@ module Gitlab end end - def gitaly_add_branch(branch_name, user, target) - gitaly_operation_client.user_create_branch(branch_name, user, target) - rescue GRPC::FailedPrecondition => ex - raise InvalidRef, ex - end - - def rugged_add_branch(branch_name, user, target) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex - end - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) OperationService.new(user, self).with_branch( branch_name, @@ -2195,8 +1957,7 @@ module Gitlab rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) + update_branch(branch, user: user, newrev: rebase_sha, oldrev: branch_sha) rebase_sha end @@ -2284,7 +2045,7 @@ module Gitlab end def gitaly_delete_refs(*ref_names) - gitaly_ref_client.delete_refs(refs: ref_names) + gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end def rugged_remove_remote(remote_name) @@ -2349,14 +2110,6 @@ module Gitlab run_git(['fetch', remote_name], env: env).last.zero? end - def gitaly_can_be_merged?(their_commit, our_commit) - !gitaly_conflicts_client(our_commit, their_commit).conflicts? - end - - def rugged_can_be_merged?(their_commit, our_commit) - !rugged.merge_commits(our_commit, their_commit).conflicts? - end - def gitlab_projects_error raise CommandError, @gitlab_projects.output end @@ -2396,19 +2149,42 @@ module Gitlab nil end - def rugged_commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count - rescue Rugged::ReferenceError - 0 - end - def rev_list_param(spec) spec == :all ? ['--all'] : spec end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end + + def build_git_cmd(*args) + object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) + + env = { 'PWD' => self.path } + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present? + + [ + env, + ::Gitlab.config.git.bin_path, + *args, + { chdir: self.path } + ] + end + + def git_diff_cmd(old_rev, new_rev) + old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev + + build_git_cmd('diff', old_rev, new_rev, '--raw') + end + + def git_cat_file_cmd + format = '%(objectname) %(objectsize) %(rest)' + build_git_cmd('cat-file', "--batch-check=#{format}") + end + + def format_git_cat_file_script + File.expand_path('../support/format-git-cat-file-input', __FILE__) + end end end end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index dc424a433fb..e35ea5762eb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -26,7 +26,7 @@ module Gitlab # When the remote repo does not have tags. if target.nil? || path.nil? Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" - return [] + break [] end name = path.split('/', 3).last @@ -35,7 +35,11 @@ module Gitlab next if name =~ /\^\{\}\Z/ target_commit = Gitlab::Git::Commit.find(self, target) - Gitlab::Git::Tag.new(self, name, target, target_commit) + Gitlab::Git::Tag.new(self, { + name: name, + target: target, + target_commit: target_commit + }) end.compact end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 38c3a55f96f..5fdad077eea 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. - module Gitlab module Git class RevList @@ -27,9 +25,10 @@ module Gitlab # # When given a block it will yield objects as a lazy enumerator so # the caller can limit work done instead of processing megabytes of data - def new_objects(require_path: nil, not_in: nil, &lazy_block) + def new_objects(options: [], require_path: nil, not_in: nil, &lazy_block) opts = { including: newrev, + options: options, excluding: not_in.nil? ? :all : not_in, require_path: require_path } @@ -37,15 +36,11 @@ module Gitlab get_objects(opts, &lazy_block) end - def all_objects(require_path: nil, &lazy_block) - get_objects(including: :all, require_path: require_path, &lazy_block) - end - - # This methods returns an array of missed references - # - # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. - def missed_ref - repository.missed_ref(oldrev, newrev).split("\n") + def all_objects(options: [], require_path: nil, &lazy_block) + get_objects(including: :all, + options: options, + require_path: require_path, + &lazy_block) end private @@ -54,8 +49,8 @@ module Gitlab repository.rev_list(args).split("\n") end - def get_objects(including: [], excluding: [], require_path: nil) - opts = { including: including, excluding: excluding, objects: true } + def get_objects(including: [], excluding: [], options: [], require_path: nil) + opts = { including: including, excluding: excluding, options: options, objects: true } repository.rev_list(opts) do |lazy_output| objects = objects_from_output(lazy_output, require_path: require_path) diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb index d3c37f82101..391f0d70583 100644 --- a/lib/gitlab/git/storage/checker.rb +++ b/lib/gitlab/git/storage/checker.rb @@ -35,7 +35,7 @@ module Gitlab def initialize(storage, logger = Rails.logger) @storage = storage config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] + @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path } @logger = logger @hostname = Gitlab::Environment.hostname diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 898bb1b65be..62427ac9cc4 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -22,13 +22,14 @@ module Gitlab def self.build(storage, hostname = Gitlab::Environment.hostname) config = Gitlab.config.repositories.storages[storage] - - if !config.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config['path'].present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) - else - new(storage, hostname) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + if !config.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) + elsif !config.legacy_disk_path.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) + else + new(storage, hostname) + end end end diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input new file mode 100755 index 00000000000..2e93c646d0f --- /dev/null +++ b/lib/gitlab/git/support/format-git-cat-file-input @@ -0,0 +1,21 @@ +#!/usr/bin/env ruby + +# This script formats the output of the `git diff <old_rev> <new_rev> --raw` +# command so it can be processed by `git cat-file` + +# We need to convert this: +# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" +# To: +# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" + +ARGF.each do |line| + _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5) + + old_blob_id.gsub!(/[^\h]/, '') + new_blob_id.gsub!(/[^\h]/, '') + + # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file + blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id + + $stdout.puts "#{blob_id} #{rest}" +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 8a8f7b051ed..e44284572fd 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,17 +1,99 @@ module Gitlab module Git class Tag < Ref - attr_reader :object_sha + extend Gitlab::EncodingHelper + + attr_reader :object_sha, :repository + + MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes + SERIALIZE_KEYS = %i[name target target_commit message].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + class << self + def get_message(repository, tag_id) + BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + tag_ids = items.map { |i| i[:tag_id] } + + messages = get_messages(repository, tag_ids) + + messages.each do |id, message| + loader.call({ repository: repository, tag_id: id }, message) + end + end + end + end + + def get_messages(repository, tag_ids) + repository.gitaly_migrate(:tag_messages) do |is_enabled| + if is_enabled + repository.gitaly_ref_client.get_tag_messages(tag_ids) + else + tag_ids.map do |id| + tag = repository.rugged.lookup(id) + message = tag.is_a?(Rugged::Commit) ? "" : tag.message + + [id, message] + end.to_h + end + end + end + end + + def initialize(repository, raw_tag) + @repository = repository + @raw_tag = raw_tag + + case raw_tag + when Hash + init_from_hash + when Gitaly::Tag + init_from_gitaly + end - def initialize(repository, name, target, target_commit, message = nil) super(repository, name, target, target_commit) + end + + def init_from_hash + raw_tag = @raw_tag.symbolize_keys + + SERIALIZE_KEYS.each do |key| + send("#{key}=", raw_tag[key]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def init_from_gitaly + @name = encode!(@raw_tag.name.dup) + @target = @raw_tag.id + @message = message_from_gitaly_tag - @message = message + if @raw_tag.target_commit.present? + @target_commit = Gitlab::Git::Commit.decorate(repository, @raw_tag.target_commit) + end end def message encode! @message end + + private + + def message_from_gitaly_tag + return @raw_tag.message.dup if full_message_fetched_from_gitaly? + + if @raw_tag.message_size > MAX_TAG_MESSAGE_DISPLAY_SIZE + '--tag message is too big' + else + self.class.get_message(@repository, target) + end + end + + def full_message_fetched_from_gitaly? + @raw_tag.message.bytesize == @raw_tag.message_size + end end end end diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb new file mode 100644 index 00000000000..1e14e8b652a --- /dev/null +++ b/lib/gitlab/git/version.rb @@ -0,0 +1,11 @@ +module Gitlab + module Git + module Version + extend Gitlab::Git::Popen + + def self.git_version + Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) + end + end + end +end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 8d82820915d..8ee46b59830 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -2,10 +2,11 @@ module Gitlab module Git class Wiki DuplicatePageError = Class.new(StandardError) + OperationError = Class.new(StandardError) - CommitDetails = Struct.new(:name, :email, :message) do + CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do def to_h - { name: name, email: email, message: message } + { user_id: user_id, username: username, name: name, email: email, message: message } end end PageBlob = Struct.new(:name) @@ -26,62 +27,38 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| - if is_enabled - gitaly_write_page(name, format, content, commit_details) - else - gollum_write_page(name, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| - if is_enabled - gitaly_delete_page(page_path, commit_details) - else - gollum_delete_page(page_path, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| - if is_enabled - gitaly_update_page(page_path, title, format, content, commit_details) - else - gollum_update_page(page_path, title, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_update_page(page_path, title, format, content, commit_details) end end def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| - if is_enabled - gitaly_get_all_pages - else - gollum_get_all_pages(limit: limit) - end + @repository.wrapped_gitaly_errors do + gitaly_get_all_pages end end def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| - if is_enabled - gitaly_find_page(title: title, version: version, dir: dir) - else - gollum_find_page(title: title, version: version, dir: dir) - end + @repository.wrapped_gitaly_errors do + gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.gitaly_migrate(:wiki_find_file) do |is_enabled| - if is_enabled - gitaly_find_file(name, version) - else - gollum_find_file(name, version) - end + @repository.wrapped_gitaly_errors do + gitaly_find_file(name, version) end end @@ -90,24 +67,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| - if is_enabled - versions = gitaly_wiki_client.page_versions(page_path, options) - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page - versions[0..slice_bound] - else - current_page = gollum_page_by_path(page_path) - - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) - end - end + versions = @repository.wrapped_gitaly_errors do + gitaly_wiki_client.page_versions(page_path, options) end + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] end def count_page_versions(page_path) @@ -129,46 +97,13 @@ module Gitlab def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.gitaly_migrate(:wiki_page_formatted_data) do |is_enabled| - if is_enabled - gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) - else - # We don't use #page because if wiki_find_page feature is enabled, we would - # get a page without formatted_data. - gollum_find_page(title: title, dir: dir, version: version)&.formatted_data - end + @repository.wrapped_gitaly_errors do + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end private - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def commits_from_page(gollum_page, options = {}) - unless options[:limit] - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - end - - @repository.log(ref: gollum_page.last_version.id, - path: gollum_page.path, - limit: options[:limit], - offset: options[:offset]) - end - - def gollum_wiki - @gollum_wiki ||= Gollum::Wiki.new(@repository.path) - end - - def gollum_page_by_path(page_path) - page_name = Gollum::Page.canonicalize_filename(page_path) - page_dir = File.split(page_path).first - - gollum_wiki.paged(page_name, page_dir) - end - def new_page(gollum_page) Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) end @@ -197,66 +132,6 @@ module Gitlab @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gollum_write_page(name, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - filename = File.basename(name) - dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - - gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir) - - nil - rescue Gollum::DuplicatePageError => e - raise Gitlab::Git::Wiki::DuplicatePageError, e.message - end - - def gollum_delete_page(page_path, commit_details) - assert_type!(commit_details, CommitDetails) - - gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) - nil - end - - def gollum_update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - page = gollum_page_by_path(page_path) - committer = Gollum::Committer.new(page.wiki, commit_details.to_h) - - # Instead of performing two renames if the title has changed, - # the update_page will only update the format and content and - # the rename_page will do anything related to moving/renaming - gollum_wiki.update_page(page, page.name, format, content, committer: committer) - gollum_wiki.rename_page(page, title, committer: committer) - committer.commit - nil - end - - def gollum_find_page(title:, version: nil, dir: nil) - if version - version = Gitlab::Git::Commit.find(@repository, version).id - end - - gollum_page = gollum_wiki.page(title, version, dir) - return unless gollum_page - - new_page(gollum_page) - end - - def gollum_find_file(name, version) - version ||= self.class.default_ref - gollum_file = gollum_wiki.file(name, version) - return unless gollum_file - - Gitlab::Git::WikiFile.new(gollum_file) - end - - def gollum_get_all_pages(limit: nil) - gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } - end - def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end @@ -288,6 +163,20 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end + + def committer_with_hooks(commit_details) + Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) + end + + def with_committer_with_hooks(commit_details, &block) + committer = committer_with_hooks(commit_details) + + yield committer + + committer.commit + + nil + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 6400089a22f..db7c29be94b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,8 +2,6 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess - include Gitlab::Utils::StrongMemoize - UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) @@ -17,7 +15,6 @@ module Gitlab deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', - account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', @@ -29,9 +26,9 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type - def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @project = project @protocol = protocol @@ -39,6 +36,7 @@ module Gitlab @namespace_path = namespace_path @project_path = project_path @redirected_path = redirected_path + @auth_result_type = auth_result_type end def check(cmd, changes) @@ -53,7 +51,7 @@ module Gitlab ensure_project_on_push!(cmd, changes) check_project_accessibility! - check_project_moved! + add_project_moved_message! check_repository_existence! case cmd @@ -78,6 +76,12 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end + def request_from_ci_build? + return false unless protocol == 'http' + + auth_result_type == :build || auth_result_type == :ci + end + def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -93,16 +97,19 @@ module Gitlab end def check_protocol! + return if request_from_ci_build? + unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" end end def check_active_user! - return if deploy_key? + return unless user - if user && !user_access.allowed? - raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] + unless user_access.allowed? + message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message + raise UnauthorizedError, message end end @@ -125,16 +132,12 @@ module Gitlab end end - def check_project_moved! + def add_project_moved_message! return if redirected_path.nil? project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) - if project_moved.permanent_redirect? - project_moved.add_message - else - raise ProjectMovedError, project_moved.message(rejected: true) - end + project_moved.add_message end def check_command_disabled!(cmd) @@ -205,6 +208,7 @@ module Gitlab def check_download_access! passed = deploy_key? || + deploy_token? || user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -219,7 +223,7 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if deploy_key + if deploy_key? unless deploy_key.can_push_to?(project) raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] end @@ -235,6 +239,11 @@ module Gitlab end def check_change_access!(changes) + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied @@ -266,6 +275,14 @@ module Gitlab actor.is_a?(DeployKey) end + def deploy_token + actor if deploy_token? + end + + def deploy_token? + actor.is_a?(DeployToken) + end + def ci? actor == :ci end @@ -273,6 +290,8 @@ module Gitlab def can_read_project? if deploy_key? deploy_key.has_access_to?(project) + elsif deploy_token? + deploy_token.has_access_to?(project) elsif user user.can?(:read_project, project) elsif ci? @@ -309,8 +328,10 @@ module Gitlab case actor when User actor + when DeployKey + nil when Key - actor.user unless actor.is_a?(DeployKey) + actor.user when :ci nil end @@ -319,6 +340,8 @@ module Gitlab def user_access @user_access ||= if ci? CiAccess.new + elsif user && request_from_ci_build? + BuildAccess.new(user, project: project) else UserAccess.new(user, project: project) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 8ca30ffc232..620362b52a9 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -83,6 +83,10 @@ module Gitlab end end + def self.random_storage + Gitlab.config.repositories.storages.keys.sample + end + def self.address(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? @@ -182,6 +186,8 @@ module Gitlab metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage + metadata.merge!(server_feature_flags) + result = { metadata: metadata } # nil timeout indicates that we should use the default @@ -200,6 +206,14 @@ module Gitlab result end + SERVER_FEATURE_FLAGS = %w[gogit_findcommit].freeze + + def self.server_feature_flags + SERVER_FEATURE_FLAGS.map do |f| + ["gitaly-feature-#{f.tr('_', '-')}", feature_enabled?(f).to_s] + end.to_h + end + def self.token(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? @@ -230,10 +244,21 @@ module Gitlab when MigrationStatus::OPT_OUT true when MigrationStatus::OPT_IN - opt_into_all_features? + opt_into_all_features? && !explicit_opt_in_required.include?(feature_name) else false end + rescue => ex + # During application startup feature lookups in SQL can fail + Rails.logger.warn "exception while checking Gitaly feature status for #{feature_name}: #{ex}" + false + end + + # We have a mechanism to let GitLab automatically opt in to all Gitaly + # features. We want to be able to exclude some features from automatic + # opt-in. This function has an override in EE. + def self.explicit_opt_in_required + [] end # opt_into_all_features? returns true when the current environment diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 456a8a1a2d6..7f2e6441f16 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -3,10 +3,6 @@ module Gitlab class CommitService include Gitlab::EncodingHelper - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -37,7 +33,7 @@ module Gitlab def diff(from, to, options = {}) from_id = case from when NilClass - EMPTY_TREE_ID + Gitlab::Git::EMPTY_TREE_ID else if from.respond_to?(:oid) # This is meant to match a Rugged::Commit. This should be impossible in @@ -50,7 +46,7 @@ module Gitlab to_id = case to when NilClass - EMPTY_TREE_ID + Gitlab::Git::EMPTY_TREE_ID else if to.respond_to?(:oid) # This is meant to match a Rugged::Commit. This should be impossible in @@ -82,7 +78,7 @@ module Gitlab def tree_entry(ref, path, limit = nil) request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, - revision: ref, + revision: encode_binary(ref), path: encode_binary(path), limit: limit.to_i ) @@ -183,6 +179,8 @@ module Gitlab end def list_commits_by_oid(oids) + return [] if oids.empty? + request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) @@ -338,6 +336,22 @@ module Gitlab signatures end + def get_commit_messages(commit_ids) + request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request) + + messages = Hash.new { |h, k| h[k] = ''.b } + current_commit_id = nil + + response.each do |rpc_message| + current_commit_id = rpc_message.commit_id if rpc_message.commit_id.present? + + messages[current_commit_id] << rpc_message.message + end + + messages + end + private def call_commit_diff(request_params, options = {}) @@ -352,7 +366,7 @@ module Gitlab end def diff_from_parent_request_params(commit, options = {}) - parent_id = commit.parent_ids.first || EMPTY_TREE_ID + parent_id = commit.parent_ids.first || Gitlab::Git::EMPTY_TREE_ID diff_between_commits_request_params(parent_id, commit.id, options) end diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index 97c13d1fdb0..c275a065bce 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -17,7 +17,7 @@ module Gitlab current_file = file_from_gitaly_header(gitaly_file.header) else - current_file.content << gitaly_file.content + current_file.raw_content << gitaly_file.content end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 831cfd1e014..e9d4bb4c4b6 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -20,7 +20,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end end @@ -35,12 +35,12 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists raise Gitlab::Git::Repository::TagExistsError end - Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + Gitlab::Git::Tag.new(@repository, response.tag) rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Repository::InvalidRef, e end @@ -56,7 +56,7 @@ module Gitlab :user_create_branch, request) if response.pre_receive_error.present? - raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) end branch = response.branch @@ -76,7 +76,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end end @@ -106,7 +106,7 @@ module Gitlab second_response = response_enum.next if second_response.pre_receive_error.present? - raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error + raise Gitlab::Git::PreReceiveError, second_response.pre_receive_error end branch_update = second_response.branch_update @@ -175,7 +175,7 @@ module Gitlab ) if response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + raise Gitlab::Git::PreReceiveError, response.pre_receive_error elsif response.git_error.presence raise Gitlab::Git::Repository::GitError, response.git_error else @@ -242,7 +242,7 @@ module Gitlab :user_commit_files, req_enum, remote_storage: start_repository.storage) if (pre_receive_error = response.pre_receive_error.presence) - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end if (index_error = response.index_error.presence) @@ -280,7 +280,7 @@ module Gitlab def handle_cherry_pick_or_revert_response(response) if response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + raise Gitlab::Git::PreReceiveError, response.pre_receive_error elsif response.commit_error.presence raise Gitlab::Git::CommitError, response.commit_error elsif response.create_tree_error.presence diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index ba6b577fd17..3ac46be6208 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -171,6 +171,22 @@ module Gitlab consume_ref_contains_sha_response(stream, :branch_names) end + def get_tag_messages(tag_ids) + request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request) + + messages = Hash.new { |h, k| h[k] = ''.b } + current_tag_id = nil + + response.each do |rpc_message| + current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present? + + messages[current_tag_id] << rpc_message.message + end + + messages + end + private def consume_refs_response(response) @@ -210,7 +226,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } + message.tags.map { |gitaly_tag| Gitlab::Git::Tag.new(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 58c356edfd1..f2d699d9dfb 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -3,6 +3,17 @@ module Gitlab class RemoteService MAX_MSG_SIZE = 128.kilobytes.freeze + def self.exists?(remote_url) + request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url) + + response = GitalyClient.call(GitalyClient.random_storage, + :remote_service, + :find_remote_repository, request, + timeout: GitalyClient.medium_timeout) + + response.exists + end + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e1bc2f9ab61..ca986434221 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -19,6 +19,11 @@ module Gitlab response.exists end + def cleanup + request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :cleanup, request) + end + def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) GitalyClient.call(@storage, :repository_service, :garbage_collect, request) @@ -45,6 +50,15 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end + def info_attributes + request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) + + response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request) + response.each_with_object("") do |message, attributes| + attributes << message.attributes + end + end + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, @@ -128,7 +142,7 @@ module Gitlab :repository_service, :is_rebase_in_progress, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) response.in_progress @@ -145,7 +159,7 @@ module Gitlab :repository_service, :is_squash_in_progress, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) response.in_progress @@ -182,41 +196,53 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.default_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) - request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo) - enum = Enumerator.new do |y| - File.open(bundle_path, 'rb') do |f| - while data = f.read(MAX_MSG_SIZE) - request.data = data + gitaly_repo_stream_request( + bundle_path, + :create_repository_from_bundle, + Gitaly::CreateRepositoryFromBundleRequest, + GitalyClient.default_timeout + ) + end - y.yield request + def restore_custom_hooks(custom_hooks_path) + gitaly_repo_stream_request( + custom_hooks_path, + :restore_custom_hooks, + Gitaly::RestoreCustomHooksRequest, + GitalyClient.default_timeout + ) + end - request = Gitaly::CreateRepositoryFromBundleRequest.new - end - end - end + def create_from_snapshot(http_url, http_auth) + request = Gitaly::CreateRepositoryFromSnapshotRequest.new( + repository: @gitaly_repo, + http_url: http_url, + http_auth: http_auth + ) GitalyClient.call( @storage, :repository_service, - :create_repository_from_bundle, - enum, + :create_repository_from_snapshot, + request, timeout: GitalyClient.default_timeout ) end @@ -257,6 +283,73 @@ module Gitlab response.license_short_name.presence end + + def calculate_checksum + request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response.checksum.presence + rescue GRPC::DataLoss => e + raise Gitlab::Git::Repository::InvalidRepository.new(e) + end + + def raw_changes_between(from, to) + request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to) + + GitalyClient.call(@storage, :repository_service, :get_raw_changes, request) + end + + def search_files_by_name(ref, query) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files) + end + + def search_files_by_content(ref, query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) + GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) + end + + private + + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we recieved an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + enum = Enumerator.new do |y| + File.open(file_path, 'rb') do |f| + while data = f.read(MAX_MSG_SIZE) + request.data = data + + y.yield request + request = request_class.new + end + end + end + + GitalyClient.call( + @storage, + :repository_service, + rpc_name, + enum, + timeout: timeout + ) + end end end end diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb new file mode 100644 index 00000000000..eb0e910665b --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + class StorageService + def initialize(storage) + @storage = storage + end + + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. + def delete_all_repositories + request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) + GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb new file mode 100644 index 00000000000..02fcb413abd --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -0,0 +1,68 @@ +module Gitlab + module GitalyClient + # This is a chokepoint that is meant to help us stop remove all places + # where production code (app, config, db, lib) touches Git repositories + # directly. + class StorageSettings + extend Gitlab::TemporarilyAllow + + DirectPathAccessError = Class.new(StandardError) + InvalidConfigurationError = Class.new(StandardError) + + INVALID_STORAGE_MESSAGE = <<~MSG.freeze + Storage is invalid because it has no `path` key. + + For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. + If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`. + MSG + + # This class will give easily recognizable NoMethodErrors + Deprecated = Class.new + + MUTEX = Mutex.new + + DISK_ACCESS_DENIED_FLAG = :deny_disk_access + ALLOW_KEY = :allow_disk_access + + # If your code needs this method then your code needs to be fixed. + def self.allow_disk_access + temporarily_allow(ALLOW_KEY) { yield } + end + + def self.disk_access_denied? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) + rescue + false # Err on the side of caution, don't break gitlab for people + end + + def initialize(storage) + raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) + raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') + + # Support a nil 'path' field because some of the circuit breaker tests use it. + @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] + + storage['path'] = Deprecated + @hash = storage + end + + def gitaly_address + @hash.fetch(:gitaly_address) + end + + def legacy_disk_path + if self.class.disk_access_denied? + raise DirectPathAccessError, "git disk access denied via the gitaly_#{DISK_ACCESS_DENIED_FLAG} feature" + end + + @legacy_disk_path + end + + private + + def method_missing(m, *args, &block) + @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index a8c6d478de8..9c19c51d412 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -3,11 +3,9 @@ module Gitlab module Util class << self def repository(repository_storage, relative_path, gl_repository) - git_object_directory = Gitlab::Git::Env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence || - Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].presence - git_alternate_object_directories = - Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']).presence || - Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']).flat_map { |d| d.split(File::PATH_SEPARATOR) } + git_env = Gitlab::Git::HookEnv.all(gl_repository) + git_object_directory = git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence + git_alternate_object_directories = Array.wrap(git_env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']) Gitaly::Repository.new( storage_name: repository_storage, @@ -23,20 +21,6 @@ module Gitlab gitaly_repository.relative_path, gitaly_repository.gl_repository) end - - def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) - if gitaly_tag.target_commit.present? - commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - repository, - Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), - gitaly_tag.id, - commit, - Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) - ) - end end end end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 0d8dd5cb8f4..2dfe055a496 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -136,7 +136,7 @@ module Gitlab wiki_file = nil response.each do |message| - next unless message.name.present? + next unless message.name.present? || wiki_file if wiki_file wiki_file.raw_data << message.raw_data @@ -200,6 +200,8 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( + user_id: commit_details.user_id, + user_name: encode_binary(commit_details.username), name: encode_binary(commit_details.name), email: encode_binary(commit_details.email), message: encode_binary(commit_details.message) diff --git a/lib/gitlab/github_import/importer/lfs_object_importer.rb b/lib/gitlab/github_import/importer/lfs_object_importer.rb new file mode 100644 index 00000000000..a88c17aaf82 --- /dev/null +++ b/lib/gitlab/github_import/importer/lfs_object_importer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LfsObjectImporter + attr_reader :lfs_object, :project + + # lfs_object - An instance of `Gitlab::GithubImport::Representation::LfsObject`. + # project - An instance of `Project`. + def initialize(lfs_object, project, _) + @lfs_object = lfs_object + @project = project + end + + def execute + Projects::LfsPointers::LfsDownloadService + .new(project) + .execute(lfs_object.oid, lfs_object.download_link) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb new file mode 100644 index 00000000000..6046e30d4ef --- /dev/null +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LfsObjectsImporter + include ParallelScheduling + + def importer_class + LfsObjectImporter + end + + def representation_class + Representation::LfsObject + end + + def sidekiq_worker_class + ImportLfsObjectWorker + end + + def collection_method + :lfs_objects + end + + def each_object_to_import + lfs_objects = Projects::LfsPointers::LfsImportService.new(project).execute + + lfs_objects.each do |object| + yield object + end + rescue StandardError => e + Rails.logger.error("The Lfs import process failed. #{e.message}") + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 49d859f9624..6b3688c4381 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -22,15 +22,22 @@ module Gitlab end def execute - if (mr_id = create_merge_request) - issuable_finder.cache_database_id(mr_id) + mr, already_exists = create_merge_request + + if mr + insert_git_data(mr, already_exists) + issuable_finder.cache_database_id(mr.id) end end # Creates the merge request and returns its ID. # # This method will return `nil` if the merge request could not be - # created. + # created, otherwise it will return an Array containing the following + # values: + # + # 1. A MergeRequest instance. + # 2. A boolean indicating if the MR already exists. def create_merge_request author_id, author_found = user_finder.author_id_for(pull_request) @@ -69,21 +76,43 @@ module Gitlab merge_request_id = GithubImport .insert_and_return_id(attributes, project.merge_requests) - merge_request = project.merge_requests.find(merge_request_id) - - # These fields are set so we can create the correct merge request - # diffs. - merge_request.source_branch_sha = pull_request.source_branch_sha - merge_request.target_branch_sha = pull_request.target_branch_sha - - merge_request.keep_around_commit - merge_request.merge_request_diffs.create - - merge_request.id + [project.merge_requests.find(merge_request_id), false] end rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this # job. In this case we'll just skip creating the merge request. + [] + rescue ActiveRecord::RecordNotUnique + # It's possible we previously created the MR, but failed when updating + # the Git data. In this case we'll just continue working on the + # existing row. + [project.merge_requests.find_by(iid: pull_request.iid), true] + end + + def insert_git_data(merge_request, already_exists = false) + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_branch_sha = pull_request.target_branch_sha + + merge_request.keep_around_commit + + # MR diffs normally use an "after_save" hook to pull data from Git. + # All of this happens in the transaction started by calling + # create/save/etc. This in turn can lead to these transactions being + # held open for much longer than necessary. To work around this we + # first save the diff, then populate it. + diff = + if already_exists + merge_request.merge_request_diffs.take || + merge_request.merge_request_diffs.build + else + merge_request.merge_request_diffs.build + end + + diff.importing = true + diff.save + diff.save_git_content end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index ab0b751fe24..01168abde6c 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -16,7 +16,8 @@ module Gitlab # Returns true if we should import the wiki for the project. def import_wiki? client.repository(project.import_source)&.has_wiki && - !project.wiki_repository_exists? + !project.wiki_repository_exists? && + Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end # Imports the repository data. @@ -55,10 +56,8 @@ module Gitlab def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" - wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - storage_path = project.repository_storage_path - gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) true rescue Gitlab::Shell::Error => e @@ -70,6 +69,10 @@ module Gitlab end end + def wiki_url + project.import_url.sub(/\.git\z/, '.wiki.git') + end + def update_clone_time project.update_column(:last_repository_updated_at, Time.zone.now) end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 6da11e6ef08..a77ac1e4fa6 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -15,6 +15,15 @@ module Gitlab true end + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore + # the visibility of prepended modules. See + # https://github.com/rspec/rspec-mocks/issues/1231 for more details. + if Rails.env.test? + def self.requires_ci_cd_setup? + raise NotImplementedError + end + end + def initialize(project) @project = project end @@ -32,7 +41,8 @@ module Gitlab Gitlab::SidekiqStatus .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - project.update_column(:import_jid, jid) + project.ensure_import_state + project.import_state&.update_column(:jid, jid) Stage::ImportRepositoryWorker .perform_async(project.id) diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb new file mode 100644 index 00000000000..debe0fa0baf --- /dev/null +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class LfsObject + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :oid, :download_link + + # Builds a lfs_object + def self.from_api_response(lfs_object) + new({ oid: lfs_object[0], download_link: lfs_object[1] }) + end + + # Builds a new lfs_object using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the raw lfs_object details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 4f7324536a0..6a181caf65d 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -19,7 +19,8 @@ module Gitlab Importer::PullRequestsImporter, Importer::IssuesImporter, Importer::DiffNotesImporter, - Importer::NotesImporter + Importer::NotesImporter, + Importer::LfsObjectsImporter ].freeze # project - The project to import the data into. @@ -41,8 +42,6 @@ module Gitlab klass.new(project, client, parallel: false).execute end - project.repository.after_import - true end end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 5482504e72e..22719e9a003 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -29,28 +29,28 @@ module Gitlab end def user - api.get("/api/v3/user").parsed + api.get("/api/v4/user").parsed end def issues(project_identifier) lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed end end def issue_comments(project_identifier, issue_id) lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed end end def project(id) - api.get("/api/v3/projects/#{id}").parsed + api.get("/api/v4/projects/#{id}").parsed end def projects lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed end end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e44d7934fda..195672f5a12 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -25,7 +25,7 @@ module Gitlab body = @formatter.author_line(issue["author"]["name"]) body += issue["description"] - comments = client.issue_comments(project_identifier, issue["id"]) + comments = client.issue_comments(project_identifier, issue["iid"]) if comments.any? body += @formatter.comments_header diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 3d0418261bb..430b8c10058 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -17,7 +17,7 @@ module Gitlab path: repo["path"], description: repo["description"], namespace_id: namespace.id, - visibility_level: repo["visibility_level"], + visibility_level: Gitlab::VisibilityLevel.level_value(repo["visibility"]), import_type: "gitlab", import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb index 624fd00367e..a53d156b41f 100644 --- a/lib/gitlab/gl_id.rb +++ b/lib/gitlab/gl_id.rb @@ -2,10 +2,14 @@ module Gitlab module GlId def self.gl_id(user) if user.present? - "user-#{user.id}" + gl_id_from_id_value(user.id) else - "" + '' end end + + def self.gl_id_from_id_value(id) + "user-#{id}" + end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index a7e055ac444..deaa14c8434 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,14 +11,15 @@ module Gitlab gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = help_page_path('shortcuts') + gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url - gon.revision = Gitlab::REVISION + gon.revision = Gitlab.revision gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path + gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.test_env = Rails.env.test? gon.suggested_label_colors = LabelsHelper.suggested_colors diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 413872d7e08..a4263369269 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -54,7 +54,11 @@ module Gitlab fingerprints = CurrentKeyChain.fingerprints_from_key(key) GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| - raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } } + raw_key.uids.each_with_object([]) do |uid, arr| + name = uid.name.force_encoding('UTF-8') + email = uid.email.force_encoding('UTF-8') + arr << { name: name, email: email.downcase } if name.valid_encoding? && email.valid_encoding? + end end end end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 1e1fdabca93..0014ce2689b 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -2,8 +2,12 @@ module Gitlab module GrapeLogging module Formatters class LogrageWithTimestamp + include Gitlab::EncodingHelper + def call(severity, datetime, _, data) time = data.delete :time + data[:params] = utf8_encode_values(data[:params]) if data.has_key?(:params) + attributes = { time: datetime.utc.iso8601(3), severity: severity, @@ -13,6 +17,19 @@ module Gitlab }.merge(data) ::Lograge.formatter.call(attributes) + "\n" end + + private + + def utf8_encode_values(data) + case data + when Hash + data.merge(data) { |k, v| utf8_encode_values(v) } + when Array + data.map { |v| utf8_encode_values(v) } + when String + encode_utf8(data) + end + end end end end diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb new file mode 100644 index 00000000000..0adac79f25a --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -0,0 +1,26 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log how much time an API request was queued by Workhorse. +module Gitlab + module GrapeLogging + module Loggers + class QueueDurationLogger < ::GrapeLogging::Loggers::Base + attr_accessor :start_time + + def before + @start_time = Time.now + end + + def parameters(request, _) + proxy_start = request.env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence + + return {} unless proxy_start && start_time + + # Time in milliseconds since gitlab-workhorse started the request + duration = (start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000).round(2) + + { 'queue_duration': duration } + end + end + end + end +end diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb new file mode 100644 index 00000000000..04a89432230 --- /dev/null +++ b/lib/gitlab/graphql.rb @@ -0,0 +1,5 @@ +module Gitlab + module Graphql + StandardGraphqlError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb new file mode 100644 index 00000000000..04f25c53e49 --- /dev/null +++ b/lib/gitlab/graphql/authorize.rb @@ -0,0 +1,21 @@ +module Gitlab + module Graphql + # Allow fields to declare permissions their objects must have. The field + # will be set to nil unless all required permissions are present. + module Authorize + extend ActiveSupport::Concern + + def self.use(schema_definition) + schema_definition.instrument(:field, Instrumentation.new) + end + + def required_permissions + @required_permissions ||= [] + end + + def authorize(*permissions) + required_permissions.concat(permissions) + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb new file mode 100644 index 00000000000..6cb8e617f62 --- /dev/null +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -0,0 +1,45 @@ +module Gitlab + module Graphql + module Authorize + class Instrumentation + # Replace the resolver for the field with one that will only return the + # resolved object if the permissions check is successful. + # + # Collections are not supported. Apply permissions checks for those at the + # database level instead, to avoid loading superfluous data from the DB + def instrument(_type, field) + field_definition = field.metadata[:type_class] + return field unless field_definition.respond_to?(:required_permissions) + return field if field_definition.required_permissions.empty? + + old_resolver = field.resolve_proc + + new_resolver = -> (obj, args, ctx) do + resolved_obj = old_resolver.call(obj, args, ctx) + checker = build_checker(ctx[:current_user], field_definition.required_permissions) + + if resolved_obj.respond_to?(:then) + resolved_obj.then(&checker) + else + checker.call(resolved_obj) + end + end + + field.redefine do + resolve(new_resolver) + end + end + + private + + def build_checker(current_user, abilities) + proc do |obj| + # Load the elements if they weren't loaded by BatchLoader yet + obj = obj.sync if obj.respond_to?(:sync) + obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) } + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb new file mode 100644 index 00000000000..2c7b64f1be9 --- /dev/null +++ b/lib/gitlab/graphql/present.rb @@ -0,0 +1,20 @@ +module Gitlab + module Graphql + module Present + extend ActiveSupport::Concern + prepended do + def self.present_using(kls) + @presenter_class = kls + end + + def self.presenter_class + @presenter_class + end + end + + def self.use(schema_definition) + schema_definition.instrument(:field, Instrumentation.new) + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb new file mode 100644 index 00000000000..1688262974b --- /dev/null +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -0,0 +1,25 @@ +module Gitlab + module Graphql + module Present + class Instrumentation + def instrument(type, field) + presented_in = field.metadata[:type_class].owner + return field unless presented_in.respond_to?(:presenter_class) + return field unless presented_in.presenter_class + + old_resolver = field.resolve_proc + + resolve_with_presenter = -> (presented_type, args, context) do + object = presented_type.object + presenter = presented_in.presenter_class.new(object, **context.to_h) + old_resolver.call(presenter, args, context) + end + + field.redefine do + resolve(resolve_with_presenter) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb new file mode 100644 index 00000000000..ffbaf65b512 --- /dev/null +++ b/lib/gitlab/graphql/variables.rb @@ -0,0 +1,37 @@ +module Gitlab + module Graphql + class Variables + Invalid = Class.new(Gitlab::Graphql::StandardGraphqlError) + + def initialize(param) + @param = param + end + + def to_h + ensure_hash(@param) + end + + private + + # Handle form data, JSON body, or a blank value + def ensure_hash(ambiguous_param) + case ambiguous_param + when String + if ambiguous_param.present? + ensure_hash(JSON.parse(ambiguous_param)) + else + {} + end + when Hash, ActionController::Parameters + ambiguous_param + when nil + {} + else + raise Invalid, "Unexpected parameter: #{ambiguous_param}" + end + rescue JSON::ParserError => e + raise Invalid.new(e) + end + end + end +end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb new file mode 100644 index 00000000000..9251ed654cd --- /dev/null +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -0,0 +1,57 @@ +module Gitlab + module HashedStorage + # Hashed Storage Migrator + # + # This is responsible for scheduling and flagging projects + # to be migrated from Legacy to Hashed storage, either one by one or in bulk. + class Migrator + BATCH_SIZE = 100 + + # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously + # + # @param [Object] start first project id for the range + # @param [Object] finish last project id for the range + def bulk_schedule(start, finish) + StorageMigratorWorker.perform_async(start, finish) + end + + # Start migration of projects from specified range + # + # Flagging a project to be migrated is a synchronous action, + # but the migration runs through async jobs + # + # @param [Object] start first project id for the range + # @param [Object] finish last project id for the range + def bulk_migrate(start, finish) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + migrate(project) + end + end + + # Flag a project to me migrated + # + # @param [Object] project that will be migrated + def migrate(project) + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + + private + + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb new file mode 100644 index 00000000000..303b05e6a9a --- /dev/null +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -0,0 +1,83 @@ +module Gitlab + module HashedStorage + module RakeHelper + def self.batch_size + ENV.fetch('BATCH', 200).to_i + end + + def self.listing_limit + ENV.fetch('LIMIT', 500).to_i + end + + def self.range_from + ENV['ID_FROM'] + end + + def self.range_to + ENV['ID_TO'] + end + + def self.range_single_item? + !range_from.nil? && range_from == range_to + end + + def self.project_id_batches(&block) + Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def self.legacy_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def self.hashed_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def self.relation_summary(relation_name, relation) + relation_count = relation.count + $stdout.puts "* Found #{relation_count} #{relation_name}".color(:green) + + relation_count + end + + def self.projects_list(relation_name, relation) + listing(relation_name, relation.with_route) do |project| + $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red) + end + end + + def self.attachments_list(relation_name, relation) + listing(relation_name, relation) do |upload| + $stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red) + end + end + + def self.listing(relation_name, relation) + relation_count = relation_summary(relation_name, relation) + return unless relation_count > 0 + + limit = listing_limit + + if relation_count > limit + $stdout.puts " ! Displaying first #{limit} #{relation_name}..." + end + + relation.find_each(batch_size: batch_size).with_index do |element, index| + yield element + + break if index + 1 >= limit + end + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index e27e16ddaf6..08495c0a59e 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -17,7 +17,7 @@ module Gitlab def check catch_timeout 10.seconds do if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s else ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index afaa59b1018..050fe7a5173 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -1,5 +1,6 @@ module Gitlab module HealthChecks + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218 class FsShardsCheck extend BaseAbstractCheck RANDOM_STRING = SecureRandom.hex(1000).freeze @@ -77,7 +78,9 @@ module Gitlab end def storage_path(storage_name) - storages_paths&.dig(storage_name, 'path') + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + storages_paths[storage_name]&.legacy_disk_path + end end # All below test methods use shell commands to perform actions on storage volumes. diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 4febb0ab430..6ab36676127 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -11,7 +11,8 @@ module Gitlab def build(user: nil, changes: {}) hook_data = { - object_kind: issuable.class.name.underscore, + object_kind: object_kind, + event_type: event_type, user: user.hook_attrs, project: issuable.project.hook_attrs, object_attributes: issuable.hook_attrs, @@ -36,6 +37,18 @@ module Gitlab private + def object_kind + issuable.class.name.underscore + end + + def event_type + if issuable.try(:confidential?) + "confidential_#{object_kind}" + else + object_kind + end + end + def issuable_builder case issuable when Issue diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 96558872a37..9aca3b0fb26 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -4,6 +4,8 @@ # calling internal IP or services. module Gitlab class HTTP + BlockedUrlError = Class.new(StandardError) + include HTTParty # rubocop:disable Gitlab/HTTParty connection_adapter ProxyHTTPConnectionAdapter diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3772ef11c7f..343487bc361 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -21,7 +21,8 @@ module Gitlab 'nl_NL' => 'Nederlands', 'tr_TR' => 'Türkçe', 'id_ID' => 'Bahasa Indonesia', - 'fil_PH' => 'Filipino' + 'fil_PH' => 'Filipino', + 'pl_PL' => 'Polski' }.freeze def available_locales diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 35d57459a3d..36fc1bcdcb7 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -3,16 +3,25 @@ module Gitlab class MetadataEntry attr_reader :entry_data + # Avoid testing too many plurals if `nplurals` was incorrectly set. + # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html + # which mentions special cases for numbers ending in 2 digits + MAX_FORMS_TO_TEST = 101 + def initialize(entry_data) @entry_data = entry_data end - def expected_plurals + def expected_forms return nil unless plural_information plural_information['nplurals'].to_i end + def forms_to_test + @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min + end + private def plural_information diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 7d3ff8c7f58..d8e7269a2c2 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,6 +1,8 @@ module Gitlab module I18n class PoLinter + include Gitlab::Utils::StrongMemoize + attr_reader :po_path, :translation_entries, :metadata_entry, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -34,7 +36,7 @@ module Gitlab end @translation_entries = entries.map do |entry_data| - Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms) end nil @@ -48,7 +50,7 @@ module Gitlab translation_entries.each do |entry| errors_for_entry = validate_entry(entry) - errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + errors[entry.msgid] = errors_for_entry if errors_for_entry.any? end errors @@ -62,6 +64,7 @@ module Gitlab validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) validate_unescaped_chars(errors, entry) + validate_translation(errors, entry) errors end @@ -81,35 +84,39 @@ module Gitlab end def validate_number_of_plurals(errors, entry) - return unless metadata_entry&.expected_plurals + return unless metadata_entry&.expected_forms return unless entry.translated? - if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals - errors << "should have #{metadata_entry.expected_plurals} "\ - "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms + errors << "should have #{metadata_entry.expected_forms} "\ + "#{'translations'.pluralize(metadata_entry.expected_forms)}" end end def validate_newlines(errors, entry) - if entry.msgid_contains_newlines? + if entry.msgid_has_multiple_lines? errors << 'is defined over multiple lines, this breaks some tooling.' end - if entry.plural_id_contains_newlines? + if entry.plural_id_has_multiple_lines? errors << 'plural is defined over multiple lines, this breaks some tooling.' end - if entry.translations_contain_newlines? + if entry.translations_have_multiple_lines? errors << 'has translations defined over multiple lines, this breaks some tooling.' end end def validate_variables(errors, entry) if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.msgid) + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) end if entry.has_plural? + validate_variables_in_message(errors, entry.plural_id, entry.plural_id) + entry.plural_translations.each do |translation| validate_variables_in_message(errors, entry.plural_id, translation) end @@ -117,41 +124,98 @@ module Gitlab end def validate_variables_in_message(errors, message_id, message_translation) - message_id = join_message(message_id) required_variables = message_id.scan(VARIABLE_REGEX) validate_unnamed_variables(errors, required_variables) - validate_translation(errors, message_id, required_variables) validate_variable_usage(errors, message_translation, required_variables) end - def validate_translation(errors, message_id, used_variables) + def validate_translation(errors, entry) + Gitlab::I18n.with_locale(locale) do + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) + end + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale}: #{e.message}" + end + + def translate_singular(entry) + used_variables = entry.msgid.scan(VARIABLE_REGEX) variables = fill_in_variables(used_variables) - begin - Gitlab::I18n.with_locale(locale) do - translated = if message_id.include?('|') - FastGettext::Translation.s_(message_id) - else - FastGettext::Translation._(message_id) - end + translation = if entry.msgid.include?('|') + FastGettext::Translation.s_(entry.msgid) + else + FastGettext::Translation._(entry.msgid) + end - translated % variables + translation % variables if used_variables.any? + end + + def translate_plural(entry) + used_variables = entry.plural_id.scan(VARIABLE_REGEX) + variables = fill_in_variables(used_variables) + + numbers_covering_all_plurals.map do |number| + translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number) + + translation % variables if used_variables.any? + end + end + + def numbers_covering_all_plurals + @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals + end + + def calculate_numbers_covering_all_plurals + required_numbers = [] + discovered_indexes = [] + counter = 0 + + while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST + index_for_count = index_for_pluralization(counter) + + unless discovered_indexes.include?(index_for_count) + discovered_indexes << index_for_count + required_numbers << counter end - # `sprintf` could raise an `ArgumentError` when invalid passing something - # other than a Hash when using named variables - # - # `sprintf` could raise `TypeError` when passing a wrong type when using - # unnamed variables - # - # FastGettext::Translation could raise `RuntimeError` (raised as a string), - # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` - # - # `FastGettext::Translation` could raise `ArgumentError` as subclassess - # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` - rescue ArgumentError, TypeError, RuntimeError => e - errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + counter += 1 + end + + required_numbers + end + + def index_for_pluralization(counter) + # This calls the C function that defines the pluralization rule, it can + # return a boolean (`false` represents 0, `true` represents 1) or an integer + # that specifies the plural form to be used for the given number + pluralization_result = Gitlab::I18n.with_locale(locale) do + FastGettext.pluralisation_rule.call(counter) + end + + case pluralization_result + when false + 0 + when true + 1 + else + pluralization_result end end @@ -172,14 +236,18 @@ module Gitlab end def validate_unnamed_variables(errors, variables) - if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } + + if unnamed_variables.any? && named_variables.any? + errors << 'is combining named variables with unnamed variables' + end + + if unnamed_variables.size > 1 errors << 'is combining multiple unnamed variables' end end def validate_variable_usage(errors, translation, required_variables) - translation = join_message(translation) - # We don't need to validate when the message is empty. # In this case we fall back to the default, which has all the the # required variables. @@ -205,10 +273,6 @@ module Gitlab def validate_flags(errors, entry) errors << "is marked #{entry.flag}" if entry.flag end - - def join_message(message) - Array(message).join - end end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index e6c95afca7e..54adb98f42d 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -11,11 +11,11 @@ module Gitlab end def msgid - entry_data[:msgid] + @msgid ||= Array(entry_data[:msgid]).join end def plural_id - entry_data[:msgid_plural] + @plural_id ||= Array(entry_data[:msgid_plural]).join end def has_plural? @@ -23,12 +23,11 @@ module Gitlab end def singular_translation - all_translations.first if has_singular_translation? + all_translations.first.to_s if has_singular_translation? end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys) - .reject(&:empty?) + @all_translations ||= translation_entries.map { |translation| Array(translation).join } end def translated? @@ -54,16 +53,16 @@ module Gitlab nplurals > 1 || !has_plural? end - def msgid_contains_newlines? - msgid.is_a?(Array) + def msgid_has_multiple_lines? + entry_data[:msgid].is_a?(Array) end - def plural_id_contains_newlines? - plural_id.is_a?(Array) + def plural_id_has_multiple_lines? + entry_data[:msgid_plural].is_a?(Array) end - def translations_contain_newlines? - all_translations.any? { |translation| translation.is_a?(Array) } + def translations_have_multiple_lines? + translation_entries.any? { |translation| translation.is_a?(Array) } end def msgid_contains_unescaped_chars? @@ -84,6 +83,11 @@ module Gitlab private + def translation_entries + @translation_entries ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + def translation_keys @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index af203ff711d..b713fa7e1cd 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.2'.freeze + VERSION = '0.2.3'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb new file mode 100644 index 00000000000..aef371d81eb --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -0,0 +1,83 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class BaseAfterExportStrategy + include ActiveModel::Validations + extend Forwardable + + StrategyError = Class.new(StandardError) + + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + + private + + attr_reader :project, :current_user + + public + + def initialize(attributes = {}) + @options = OpenStruct.new(attributes) + + self.class.instance_eval do + def_delegators :@options, *attributes.keys + end + end + + def execute(current_user, project) + return unless project&.export_project_path + + @project = project + @current_user = current_user + + if invalid? + log_validation_errors + + return + end + + create_or_update_after_export_lock + strategy_execute + + true + rescue => e + project.import_export_shared.error(e) + false + ensure + delete_after_export_lock + end + + def to_json(options = {}) + @options.to_h.merge!(klass: self.class.name).to_json + end + + def self.lock_file_path(project) + return unless project&.export_path + + File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + end + + protected + + def strategy_execute + raise NotImplementedError + end + + private + + def create_or_update_after_export_lock + FileUtils.touch(self.class.lock_file_path(project)) + end + + def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) + end + + def log_validation_errors + errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb new file mode 100644 index 00000000000..4371a7eff56 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class DownloadNotificationStrategy < BaseAfterExportStrategy + private + + def strategy_execute + notification_service.project_exported(project, current_user) + end + + def notification_service + @notification_service ||= NotificationService.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb new file mode 100644 index 00000000000..938664a95a1 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -0,0 +1,61 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class WebUploadStrategy < BaseAfterExportStrategy + PUT_METHOD = 'PUT'.freeze + POST_METHOD = 'POST'.freeze + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + + validates :url, url: true + + validate do + unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase) + errors.add(:http_method, INVALID_HTTP_METHOD) + end + end + + def initialize(url:, http_method: PUT_METHOD) + super + end + + protected + + def strategy_execute + handle_response_error(send_file) + + project.remove_exported_project_file + end + + def handle_response_error(response) + unless response.success? + error_code = response.dig('Error', 'Code') || response.code + error_message = response.dig('Error', 'Message') || response.message + + raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + end + end + + private + + def send_file + export_file = File.open(project.export_project_path) + + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + ensure + export_file.close if export_file + end + + def send_file_options(export_file) + { + body_stream: export_file, + headers: headers + } + end + + def headers + { 'Content-Length' => File.size(project.export_project_path).to_s } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb new file mode 100644 index 00000000000..7eabcae2380 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class AfterExportStrategyBuilder + StrategyNotFoundError = Class.new(StandardError) + + def self.build!(strategy_klass, attributes = {}) + return default_strategy.new unless strategy_klass + + attributes ||= {} + klass = strategy_klass.constantize rescue nil + + unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy + raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + end + + klass.new(**attributes.symbolize_keys) + end + + def self.default_strategy + AfterExportStrategies::DownloadNotificationStrategy + end + end + end +end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 34169319b26..7c9fc5c15bb 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -7,14 +7,15 @@ module Gitlab new(*args).clean end - def initialize(relation_hash:, relation_class:) + def initialize(relation_hash:, relation_class:, excluded_keys: []) @relation_hash = relation_hash @relation_class = relation_class + @excluded_keys = excluded_keys end def clean @relation_hash.reject do |key, _value| - prohibited_key?(key) || !@relation_class.attribute_method?(key) + prohibited_key?(key) || !@relation_class.attribute_method?(key) || excluded_key?(key) end.except('id') end @@ -23,6 +24,12 @@ module Gitlab def prohibited_key?(key) key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) end + + def excluded_key?(key) + return false if @excluded_keys.empty? + + @excluded_keys.include?(key) + end end end end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 56042ddecbf..0c8fda07294 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -32,6 +32,10 @@ module Gitlab @methods[key].nil? ? {} : { methods: @methods[key] } end + def find_excluded_keys(klass_name) + @excluded_attributes[klass_name.to_sym]&.map(&:to_s) || [] + end + private def find_attributes_only(value) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 4bdd01f5e94..da3667faf7a 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,9 +27,8 @@ project_tree: - :releases - project_members: - :user - - lfs_file_locks: - - :user - merge_requests: + - :metrics - notes: - :author - events: @@ -66,6 +65,7 @@ project_tree: - :project_feature - :custom_attributes - :project_badges + - :ci_cd_settings # Only include the following attributes for the models specified. included_attributes: @@ -75,6 +75,8 @@ included_attributes: - :username author: - :name + ci_cd_settings: + - :group_runners_enabled # Do not include the following attributes for the models specified. excluded_attributes: @@ -97,14 +99,14 @@ excluded_attributes: - :import_jid - :created_at - :updated_at - - :import_jid - - :import_jid - :id - :star_count - :last_activity_at - :last_repository_updated_at - :last_repository_check_at - :storage_version + - :remote_mirror_available_overridden + - :description_html snippets: - :expired_at merge_request_diff: @@ -124,6 +126,8 @@ excluded_attributes: - :trace - :token - :when + - :artifacts_file + - :artifacts_metadata push_event_payload: - :event_id project_badges: @@ -144,8 +148,6 @@ methods: - :diff_head_sha - :source_branch_sha - :target_branch_sha - project: - - :description_html events: - :action push_event_payload: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c38df9102eb..63cab07324a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,9 @@ module Gitlab module ImportExport class Importer + include Gitlab::Allowable + include Gitlab::Utils::StrongMemoize + def self.imports_repository? true end @@ -13,17 +16,24 @@ module Gitlab end def execute - if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end - + rescue => e + raise Projects::ImportService::Error.new(e.message) + ensure remove_import_file end private + def restorers + [repo_restorer, wiki_restorer, project_tree, avatar_restorer, + uploads_restorer, lfs_restorer, statistics_restorer] + end + def import_file Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, shared: @shared) @@ -60,6 +70,14 @@ module Gitlab Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) end + def lfs_restorer + Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def statistics_restorer + Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def path_with_namespace File.join(@project.namespace.full_path, @project.path) end @@ -75,6 +93,33 @@ module Gitlab def remove_import_file FileUtils.rm_rf(@archive_file) end + + def overwrite_project + project = project_tree.restored_project + + return unless can?(@current_user, :admin_namespace, project.namespace) + + if overwrite_project? + ::Projects::OverwriteProjectService.new(project, @current_user) + .execute(project_to_overwrite) + end + + true + end + + def original_path + @project.import_data&.data&.fetch('original_path', nil) + end + + def overwrite_project? + original_path.present? && project_to_overwrite.present? + end + + def project_to_overwrite + strong_memoize(:project_to_overwrite) do + Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + end + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb new file mode 100644 index 00000000000..b28c3c161b7 --- /dev/null +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -0,0 +1,43 @@ +module Gitlab + module ImportExport + class LfsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true if lfs_file_paths.empty? + + lfs_file_paths.each do |file_path| + link_or_create_lfs_object!(file_path) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def link_or_create_lfs_object!(path) + size = File.size(path) + oid = LfsObject.calculate_oid(path) + + lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) + lfs_object.file = File.open(path) unless lfs_object.file&.exists? + + @project.all_lfs_objects << lfs_object + end + + def lfs_file_paths + @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") + end + + def lfs_storage_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb new file mode 100644 index 00000000000..29410e2331c --- /dev/null +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -0,0 +1,55 @@ +module Gitlab + module ImportExport + class LfsSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + @project.all_lfs_objects.each do |lfs_object| + save_lfs_object(lfs_object) + end + + true + rescue => e + @shared.error(e) + + false + end + + private + + def save_lfs_object(lfs_object) + if lfs_object.local_store? + copy_file_for_lfs_object(lfs_object) + else + download_file_for_lfs_object(lfs_object) + end + end + + def download_file_for_lfs_object(lfs_object) + destination = destination_path_for_object(lfs_object) + mkdir_p(File.dirname(destination)) + + File.open(destination, 'w') do |file| + IO.copy_stream(URI.parse(lfs_object.file.url).open, file) + end + end + + def copy_file_for_lfs_object(lfs_object) + copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) + end + + def destination_path_for_object(lfs_object) + File.join(lfs_export_path, lfs_object.oid) + end + + def lfs_export_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 8f5bb8f9597..4eb67fbe11e 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -77,24 +77,33 @@ module Gitlab end def default_relation_list - Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] end end def restore_project - params = project_params + @project.update_columns(project_params) + @project + end + + def project_params + @project_params ||= begin + attrs = json_params.merge(override_params) - if params[:description].present? - params[:description_html] = nil + # Cleaning all imported and overridden params + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs, + relation_class: Project, + excluded_keys: excluded_keys_for_relation(:project)) end + end - @project.update_columns(params) - @project + def override_params + @override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} end - def project_params - @tree_hash.reject do |key, value| + def json_params + @json_params ||= @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations value.respond_to?(:each) && !Project.column_names.include?(key) end @@ -164,7 +173,8 @@ module Gitlab relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), members_mapper: members_mapper, user: @user, - project: @restored_project) + project: @restored_project, + excluded_keys: excluded_keys_for_relation(relation)) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first @@ -181,6 +191,14 @@ module Gitlab relation_hash.merge(params) end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def excluded_keys_for_relation(relation) + @reader.attributes_finder.find_excluded_keys(relation) + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index eb7f5120592..e621c40fc7a 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class Reader - attr_reader :tree + attr_reader :tree, :attributes_finder def initialize(shared:) @shared = shared diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 791a54e1b69..c5cf290f191 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -17,9 +17,11 @@ module Gitlab auto_devops: :project_auto_devops, label: :project_label, custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge' }.freeze + project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', + ci_cd_settings: 'ProjectCiCdSetting' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -27,7 +29,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze @@ -35,13 +37,30 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) + def self.relation_class(relation_name) + # There are scenarios where the model is pluralized (e.g. + # MergeRequest::Metrics), and we don't want to force it to singular + # with #classify. + relation_name.to_s.classify.constantize + rescue NameError + relation_name.to_s.constantize + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) @relation_name = OVERRIDES[relation_sym] || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @user = user @project = project @imported_object_retries = 0 + + # Remove excluded keys from relation_hash + # We don't do this in the parsed_relation_hash because of the 'transformed attributes' + # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, + # in the create method that attribute is renamed to diff. And because diff is an excluded key, + # if we clean the excluded keys in the parsed_relation_hash, it will be removed + # from the object attributes and the export will fail. + @relation_hash.except!(*excluded_keys) end # Creates an object from an actual model with name "relation_sym" with params from @@ -186,7 +205,7 @@ module Gitlab end def relation_class - @relation_class ||= @relation_name.to_s.classify.constantize + @relation_class ||= self.class.relation_class(@relation_name) end def imported_object diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 695462c7dd2..0c224bd1971 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -26,10 +26,6 @@ module Gitlab @shared.error(e) false end - - def path_to_repo - @project.repository.path_to_repo - end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 3d3d998a6a3..6d7c36ce38b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -22,7 +22,7 @@ module Gitlab def error(error) error_out(error.message, caller[0].dup) - @errors << error.message + add_error_message(error.message) # Debug: if error.backtrace @@ -32,6 +32,14 @@ module Gitlab end end + def add_error_message(error_message) + @errors << error_message + end + + def after_export_in_progress? + File.exist?(after_export_lock_file) + end + private def relative_path @@ -45,6 +53,10 @@ module Gitlab def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb new file mode 100644 index 00000000000..bcdd9c12c85 --- /dev/null +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + class StatisticsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + @project.statistics.refresh! + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 5fa2e101e29..2fd62c0fc7b 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -22,12 +22,8 @@ module Gitlab "project.wiki.bundle" end - def path_to_repo - @wiki.repository.path_to_repo - end - def wiki_repository_exists? - File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + @wiki.repository.exists? && !@wiki.repository.empty? end end end diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index 3e54456e936..4e611e7f16c 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -9,6 +9,7 @@ module Gitlab end def author_line(author) + author ||= "Anonymous" "*Created by: #{author}*\n\n" end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index c9122a23568..d323cb9dadf 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -57,7 +57,7 @@ module Gitlab regex = Regexp.escape(wildcard_address) regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') - Regexp.new(regex).freeze + Regexp.new(/\A#{regex}\z/).freeze end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 6e4df05aa7e..3d778da90c7 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -15,6 +15,9 @@ module Gitlab def generate_script <<~HEREDOC set -eo pipefail + ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2) + echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories + echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories apk add -U ca-certificates openssl >/dev/null wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null mv /tmp/linux-amd64/helm /usr/bin/ diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 30af3e97b4a..d2133a6d65b 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -2,11 +2,12 @@ module Gitlab module Kubernetes module Helm class InstallCommand < BaseCommand - attr_reader :name, :chart, :repository, :values + attr_reader :name, :chart, :version, :repository, :values - def initialize(name, chart:, values:, repository: nil) + def initialize(name, chart:, values:, version: nil, repository: nil) @name = name @chart = chart + @version = version @values = values @repository = repository end @@ -39,9 +40,13 @@ module Gitlab def script_command <<~HEREDOC - helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null + helm install #{chart} --name #{name}#{optional_version_flag} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null HEREDOC end + + def optional_version_flag + " --version #{version}" if version + end end end end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 0526ef9eb13..b04d678cf98 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -78,7 +78,8 @@ module Gitlab def handle_errors return unless errors.any? - project.update_column(:import_error, { + project.ensure_import_state + project.import_state&.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) @@ -259,7 +260,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 3ce245a8050..5e96eb16754 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -35,7 +35,10 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility + visibility_level = repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC + visibility_level = Gitlab::CurrentSettings.default_project_visibility if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level) + + visibility_level end # diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index d12ba0ec176..d41a855bff1 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -25,6 +25,14 @@ module Gitlab end end + def reset_registry! + clear_memoization(:registry) + + REGISTRY_MUTEX.synchronize do + ::Prometheus::Client.reset! + end + end + def registry strong_memoize(:registry) do REGISTRY_MUTEX.synchronize do diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5a0f7f28fc8..ad97632e4eb 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -16,12 +16,6 @@ module Gitlab @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) - - if Gitlab::Metrics.mri? - require 'allocations' - - Allocations.start - end end def sample diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4e1ea62351f..7b2b3bedf04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -20,39 +20,29 @@ module Gitlab {} end - def initialize(interval) - super(interval) - - if Metrics.mri? - require 'allocations' - - Allocations.start - end - end - def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -60,11 +50,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index db8bdde74b2..47b4af5d649 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -4,6 +4,8 @@ require 'prometheus/client/rack/exporter' module Gitlab module Metrics class SidekiqMetricsExporter < Daemon + LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') + def enabled? Gitlab::Metrics.metrics_folder_present? && settings.enabled end @@ -17,7 +19,13 @@ module Gitlab attr_reader :server def start_working - @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + logger = WEBrick::Log.new(LOG_FILENAME) + access_log = [ + [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] + ] + + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address, + Logger: logger, AccessLog: access_log) server.mount "/", Rack::Handler::WEBrick, rack_app server.start end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 4b3e8d0a6a0..38f119cf06d 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -20,7 +20,7 @@ module Gitlab define_histogram :gitlab_sql_duration_seconds do docstring 'SQL time' base_labels Transaction::BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end def current_transaction diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index f3e48083c19..9f903e96585 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -140,7 +140,7 @@ module Gitlab define_histogram :gitlab_transaction_duration_seconds do docstring 'Transaction duration' base_labels BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end define_histogram :gitlab_transaction_allocated_memory_bytes do diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 89ff02a96d6..723ca576aab 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -3,18 +3,7 @@ module Gitlab class WebTransaction < Transaction CONTROLLER_KEY = 'action_controller.instance'.freeze ENDPOINT_KEY = 'api.endpoint'.freeze - - CONTENT_TYPES = { - 'text/html' => :html, - 'text/plain' => :txt, - 'application/json' => :json, - 'text/js' => :js, - 'application/atom+xml' => :atom, - 'image/png' => :png, - 'image/jpeg' => :jpeg, - 'image/gif' => :gif, - 'image/svg+xml' => :svg - }.freeze + ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) def initialize(env) super() @@ -40,9 +29,17 @@ module Gitlab controller = @env[CONTROLLER_KEY] action = "#{controller.action_name}" - suffix = CONTENT_TYPES[controller.content_type] - if suffix && suffix != :html + # Devise exposes a method called "request_format" that does the below. + # However, this method is not available to all controllers (e.g. certain + # Doorkeeper controllers). As such we use the underlying code directly. + suffix = controller.request.format.try(:ref).to_s + + # Sometimes the request format is set to silly data such as + # "application/xrds+xml" or actual URLs. To prevent such values from + # increasing the cardinality of our metrics, we limit the number of + # possible suffixes. + if suffix && ALLOWED_SUFFIXES.include?(suffix) action += ".#{suffix}" end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index d4c54049b74..a5f5d719cc1 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -82,7 +82,7 @@ module Gitlab end def open_file(path, name) - ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream') + ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream') end end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 43921a8c1c0..fd5de73c526 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -5,7 +5,7 @@ module Gitlab def initialize(*collections, per_page: nil) raise ArgumentError.new('Only 2 collections are supported') if collections.size != 2 - @per_page = per_page || Kaminari.config.default_per_page + @per_page = (per_page || Kaminari.config.default_per_page).to_i @first_collection, @second_collection = collections end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 1d9a5d1a20a..d09bce642b0 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -3,18 +3,15 @@ module Gitlab module_function def retry_lock(subject, retries = 100, &block) - loop do - begin - ActiveRecord::Base.transaction do - return yield(subject) - end - rescue ActiveRecord::StaleObjectError - retries -= 1 - raise unless retries >= 0 - - subject.reload - end + ActiveRecord::Base.transaction do + yield(subject) end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + + subject.reload + retry end alias_method :retry_optimistic_lock, :retry_lock diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb new file mode 100644 index 00000000000..7b358a3bd1b --- /dev/null +++ b/lib/gitlab/pages_client.rb @@ -0,0 +1,117 @@ +module Gitlab + class PagesClient + class << self + attr_reader :certificate, :token + + def call(service, rpc, request, timeout: nil) + kwargs = request_kwargs(timeout) + stub(service).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + end + + # This function is not thread-safe. Call it from an initializer only. + def read_or_create_token + @token = read_token + rescue Errno::ENOENT + # TODO: uncomment this when omnibus knows how to write the token file for us + # https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2466 + # + # write_token(SecureRandom.random_bytes(64)) + # + # # Read from disk in case someone else won the race and wrote the file + # # before us. If this fails again let the exception bubble up. + # @token = read_token + end + + # This function is not thread-safe. Call it from an initializer only. + def load_certificate + cert_path = config.certificate + return unless cert_path.present? + + @certificate = File.read(cert_path) + end + + def ping + request = Grpc::Health::V1::HealthCheckRequest.new + call(:health_check, :check, request, timeout: 5.seconds) + end + + private + + def request_kwargs(timeout) + encoded_token = Base64.strict_encode64(token.to_s) + metadata = { + 'authorization' => "Bearer #{encoded_token}" + } + + result = { metadata: metadata } + + return result unless timeout + + # Do not use `Time.now` for deadline calculation, since it + # will be affected by Timecop in some tests, but grpc's c-core + # uses system time instead of timecop's time, so tests will fail + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will + # circumvent timecop + deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + result[:deadline] = deadline + + result + end + + def stub(name) + stub_class(name).new(address, grpc_creds) + end + + def stub_class(name) + if name == :health_check + Grpc::Health::V1::Health::Stub + else + # TODO use pages namespace + Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + end + end + + def address + addr = config.address + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + addr + end + + def grpc_creds + if address.start_with?('unix:') + :this_channel_is_insecure + elsif @certificate + GRPC::Core::ChannelCredentials.new(@certificate) + else + # Use system certificate pool + GRPC::Core::ChannelCredentials.new + end + end + + def config + Gitlab.config.pages.admin + end + + def read_token + File.read(token_path) + end + + def token_path + Rails.root.join('.gitlab_pages_secret').to_s + end + + def write_token(new_token) + Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| + f.write(new_token) + f.close + File.link(f.path, token_path) + end + rescue Errno::EACCES => ex + # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672 + Rails.logger.error("Could not write pages admin token file: #{ex}") + rescue Errno::EEXIST + # Another process wrote the token file concurrently with us. Use their token, not ours. + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 4dc38aae61e..61653044433 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -31,6 +31,7 @@ module Gitlab deploy.html explore favicon.ico + favicon.png files groups health_check diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 6c2b2036074..92a308a12dc 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -5,6 +5,7 @@ module Gitlab def self.enabled?(user = nil) return true if Rails.env.development? + return true if user&.admin? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 18540e64d4c..ecff6ab5d5e 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -11,6 +11,7 @@ module Gitlab lib/gitlab/etag_caching/ lib/gitlab/metrics/ lib/gitlab/middleware/ + ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ lib/gitlab/request_profiler/ lib/gitlab/profiler.rb @@ -98,11 +99,7 @@ module Gitlab super - backtrace = Rails.backtrace_cleaner.clean(caller) - - backtrace.each do |caller_line| - next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) - + Gitlab::Profiler.clean_backtrace(caller).each do |caller_line| stripped_caller_line = caller_line.sub("#{Rails.root}/", '') super(" ↳ #{stripped_caller_line}") @@ -112,6 +109,12 @@ module Gitlab end end + def self.clean_backtrace(backtrace) + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| + line.match(Regexp.union(IGNORE_BACKTRACES)) + end + end + def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging original_activerecord_logger = ActiveRecord::Base.logger diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 390efda326a..38bdc61d8ab 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -59,7 +59,7 @@ module Gitlab startline = 0 result.each_line.each_with_index do |line, index| - prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>.*)\x00(?<startline>\d+)\x00/)&.tap do |matches| + prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches| ref = matches[:ref] filename = matches[:filename] startline = matches[:startline] @@ -106,7 +106,8 @@ module Gitlab project_wiki = ProjectWiki.new(project) unless project_wiki.empty? - project_wiki.search_files(query) + ref = repository_ref || project.wiki.default_branch + Gitlab::WikiFileFinder.new(project, ref).find(query) else [] end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index ae136202f0c..08f6a54776f 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -25,9 +25,9 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, gemfile, rakefile, and .gitlab-ci.yml file, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), - ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw, pom.xml, and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), - ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') + ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), + ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw and pom.xml to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), + ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') ].freeze class << self diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index aad76e335af..f5879de1e94 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -79,7 +79,7 @@ module Gitlab def common_query_context(environment, timeframe_start:, timeframe_end:) base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, - kube_namespace: environment.project.deployment_platform&.actual_namespace || '', + kube_namespace: environment.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} }) end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index c70d6f4cd84..d682289b632 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -10,8 +10,12 @@ module Gitlab class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - if !allow_local_requests? && blocked_url? - raise URI::InvalidURIError + unless allow_local_requests? + begin + Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + end end super @@ -19,10 +23,6 @@ module Gitlab private - def blocked_url? - Gitlab::UrlBlocker.blocked_url?(uri, allow_private_networks: false) - end - def allow_local_requests? options.fetch(:allow_local_requests, allow_settings_local_requests?) end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 4c83581c4b1..3c4ff5d1928 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -4,7 +4,7 @@ module Gitlab attach_to :active_record def sql(event) - unless event.payload[:name] == 'CACHE' + unless event.payload.fetch(:cached, event.payload[:name] == 'CACHE') Transaction.current&.increment end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 075ff91700c..30c6806b68e 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -39,7 +39,7 @@ module Gitlab content.delete!("\r") content.gsub!(commands_regex) do if $~[:cmd] - commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' else $~[0] @@ -102,14 +102,14 @@ module Gitlab # /close ^\/ - (?<cmd>#{Regexp.union(names)}) + (?<cmd>#{Regexp.new(Regexp.union(names).source, Regexp::IGNORECASE)}) (?: [ ] (?<arg>[^\n]*) )? (?:\n|$) ) - }mx + }mix end def perform_substitutions(content, commands) @@ -120,7 +120,7 @@ module Gitlab end substitution_definitions.each do |substitution| - match_data = substitution.match(content) + match_data = substitution.match(content.downcase) if match_data command = [substitution.name.to_s] command << match_data[1] unless match_data[1].empty? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 032c49ed159..688056e5d73 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -15,7 +15,7 @@ module Gitlab return unless content all_names.each do |a_name| - content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) end content end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 10bec7a90da..e5a0fdae7ef 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -5,6 +5,8 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze + USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 79265cf952d..4888184403c 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -4,7 +4,8 @@ module Gitlab def self.parse(repo_path) wiki = false - project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) + project_path = repo_path.sub(/\.git\z/, '').sub(%r{\A/}, '') + project, was_redirected = find_project(project_path) if project_path.end_with?('.wiki') && project.nil? @@ -17,22 +18,6 @@ module Gitlab [project, wiki, redirected_path] end - def self.strip_storage_path(repo_path, fail_on_not_found: true) - result = repo_path - - storage = Gitlab.config.repositories.storages.values.find do |params| - repo_path.start_with?(params['path']) - end - - if storage - result = result.sub(storage['path'], '') - elsif fail_on_not_found - raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") - end - - result.sub(%r{\A/*}, '') - end - def self.find_project(project_path) project = Project.find_by_full_path(project_path, follow_redirects: true) was_redirected = project && project.full_path.casecmp(project_path) != 0 diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index ccfe0d6bed3..a502ad8a541 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -5,7 +5,7 @@ module Gitlab module RequestForgeryProtection class Controller < ActionController::Base - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true rescue_from ActionController::InvalidAuthenticityToken do |e| logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb new file mode 100644 index 00000000000..23595f23f01 --- /dev/null +++ b/lib/gitlab/search/parsed_query.rb @@ -0,0 +1,23 @@ +module Gitlab + module Search + class ParsedQuery + attr_reader :term, :filters + + def initialize(term, filters) + @term = term + @filters = filters + end + + def filter_results(results) + filters = @filters.reject { |filter| filter[:matcher].nil? } + return unless filters + + results.select do |result| + filters.all? do |filter| + filter[:matcher].call(filter, result) + end + end + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb new file mode 100644 index 00000000000..8583bce7792 --- /dev/null +++ b/lib/gitlab/search/query.rb @@ -0,0 +1,55 @@ +module Gitlab + module Search + class Query < SimpleDelegator + def initialize(query, filter_opts = {}, &block) + @raw_query = query.dup + @filters = [] + @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts) + + self.instance_eval(&block) if block_given? + + @query = Gitlab::Search::ParsedQuery.new(*extract_filters) + # set the ParsedQuery as our default delegator thanks to SimpleDelegator + super(@query) + end + + private + + def filter(name, **attributes) + filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + + @filters << filter + end + + def filter_options(**options) + @filter_options.merge!(options) + end + + def extract_filters + fragments = [] + + filters = @filters.each_with_object([]) do |filter, parsed_filters| + match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + next unless match + + input = match.split(':')[1..-1].join + next if input.empty? + + filter[:value] = parse_filter(filter, input) + filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') + fragments << match + + parsed_filters << filter + end + + query = (@raw_query.split - fragments).join(' ') + + [query, filters] + end + + def parse_filter(filter, input) + filter[:parser].call(input) + end + end + end +end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 4a22fc80f75..6381e94c1d2 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -18,6 +18,25 @@ module Gitlab end end + # This can be used for investigating exceptions that can be recovered from in + # code. The exception will still be raised in development and test + # environments. + # + # That way we can track down these exceptions with as much information as we + # need to resolve them. + # + # Provide an issue URL for follow up. + def self.track_exception(exception, issue_url: nil, extra: {}) + if enabled? + extra[:issue_url] = issue_url if issue_url + context # Make sure we've set everything we know in the context + + Raven.capture_exception(exception, extra: extra) + end + + raise exception if should_raise? + end + def self.program_context if Sidekiq.server? 'sidekiq' @@ -25,5 +44,9 @@ module Gitlab 'rails' end end + + def self.should_raise? + Rails.env.development? || Rails.env.test? + end end end diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb index 9c92b83dddc..6bb00d8ae21 100644 --- a/lib/gitlab/serializer/pagination.rb +++ b/lib/gitlab/serializer/pagination.rb @@ -17,8 +17,6 @@ module Gitlab end end - private - # Methods needed by `API::Helpers::Pagination` # diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 07d7c91cb5d..b2d75aac1d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,7 +24,10 @@ module Gitlab address = val['gitaly_address'] end - storages << { name: key, path: val['path'] } + # https://gitlab.com/gitlab-org/gitaly/issues/1238 + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + storages << { name: key, path: val.legacy_disk_path } + end end if Rails.env.test? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 3a8f5826818..5cedd9e84c2 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,5 +1,4 @@ -# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. -# SSH key operations are not part of Gitaly so will never be migrated. +# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated. require 'securerandom' @@ -65,11 +64,11 @@ module Gitlab # Init new repository # - # storage - project's storage name + # storage - the shard key # name - project disk path # # Ex. - # create_repository("/path/to/storage", "gitlab/gitlab-ci") + # create_repository("default", "gitlab/gitlab-ci") # def create_repository(storage, name) relative_path = name.dup @@ -82,7 +81,7 @@ module Gitlab repository.gitaly_repository_client.create_repository true else - repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path) Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) end end @@ -93,12 +92,12 @@ module Gitlab # Import repository # - # storage - project's storage path + # storage - project's storage name # name - project disk path # url - URL to import from # # Ex. - # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") + # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) @@ -106,10 +105,17 @@ module Gitlab raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end - # The timeout ensures the subprocess won't hang forever - cmd = gitlab_projects(storage, "#{name}.git") - success = cmd.import_project(url, git_timeout) + relative_path = "#{name}.git" + cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + GitalyGitlabProjects.new(storage, relative_path) + else + # The timeout ensures the subprocess won't hang forever + gitlab_projects(storage, relative_path) + end + end + success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success success @@ -131,8 +137,7 @@ module Gitlab if is_enabled repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else - storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) + local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -147,8 +152,6 @@ module Gitlab # # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) return false if path.empty? || new_path.empty? @@ -156,18 +159,18 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage path - # forked_from_disk_path - project disk path - # forked_to_storage - forked-to project's storage path - # forked_to_disk_path - forked project disk path + # forked_from_storage - forked-from project's storage name + # forked_from_disk_path - project disk relative path + # forked_to_storage - forked-to project's storage name + # forked_to_disk_path - forked project disk relative path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 + # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) - gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") - .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") + forked_from_relative_path = "#{forked_from_disk_path}.git" + fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] + + GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -179,8 +182,6 @@ module Gitlab # # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) return false if name.empty? @@ -292,20 +293,10 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("/path/to/storage", "gitlab") + # add_namespace("default", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_namespace_client(storage).add(name) - else - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) - end - end - rescue Errno::EEXIST => e - Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + Gitlab::GitalyClient::NamespaceService.new(storage).add(name) rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end @@ -314,17 +305,10 @@ module Gitlab # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("/path/to/storage", "gitlab") + # rm_namespace("default", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_namespace_client(storage).remove(name) - else - FileUtils.rm_r(full_path(storage, name), force: true) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end @@ -336,16 +320,7 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_namespace_client(storage).rename(old_name, new_name) - else - return false if exists?(storage, new_name) || !exists?(storage, old_name) - - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) rescue GRPC::InvalidArgument false end @@ -370,16 +345,8 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_namespace_client(storage).exists?(dir_name) - else - File.exist?(full_path(storage, dir_name)) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) end protected @@ -399,7 +366,7 @@ module Gitlab def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(storage, dir_name) + File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name) end def gitlab_shell_projects_path @@ -420,16 +387,16 @@ module Gitlab private - def gitlab_projects(shard_path, disk_path) + def gitlab_projects(shard_name, disk_path) Gitlab::Git::GitlabProjects.new( - shard_path, + shard_name, disk_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? @@ -442,7 +409,7 @@ module Gitlab end end - cmd = gitlab_projects(storage_path, repository_relative_path) + cmd = gitlab_projects(storage_name, repository_relative_path) success = cmd.fetch_remote(remote, git_timeout, vars) @@ -476,14 +443,6 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end - def gitaly_namespace_client(storage_path) - storage, _value = Gitlab.config.repositories.storages.find do |storage, value| - value['path'] == storage_path - end - - Gitlab::GitalyClient::NamespaceService.new(storage) - end - def git_timeout Gitlab.config.gitlab_shell.git_timeout end @@ -495,5 +454,39 @@ module Gitlab # need to do the same here... raise Error, e end + + class GitalyGitlabProjects + attr_reader :shard_name, :repository_relative_path, :output + + def initialize(shard_name, repository_relative_path) + @shard_name = shard_name + @repository_relative_path = repository_relative_path + @output = '' + end + + def import_project(source, _timeout) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output = e.message + false + end + + def fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end + + def logger + Rails.logger + end + end end end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb new file mode 100644 index 00000000000..98f8222fd03 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module SidekiqLogging + class JSONFormatter + def call(severity, timestamp, progname, data) + output = { + severity: severity, + time: timestamp.utc.iso8601(3) + } + + case data + when String + output[:message] = data + when Hash + output.merge!(data) + end + + output.to_json + "\n" + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb new file mode 100644 index 00000000000..9a89ae70b98 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -0,0 +1,96 @@ +module Gitlab + module SidekiqLogging + class StructuredLogger + START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze + DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + + def call(job, queue) + started_at = current_time + base_payload = parse_job(job) + + Sidekiq.logger.info log_job_start(started_at, base_payload) + + yield + + Sidekiq.logger.info log_job_done(started_at, base_payload) + rescue => job_exception + Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + + raise + end + + private + + def base_message(payload) + "#{payload['class']} JID-#{payload['jid']}" + end + + def log_job_start(started_at, payload) + payload['message'] = "#{base_message(payload)}: start" + payload['job_status'] = 'start' + + payload + end + + def log_job_done(started_at, payload, job_exception = nil) + payload = payload.dup + payload['duration'] = elapsed(started_at) + payload['completed_at'] = Time.now.utc + + message = base_message(payload) + + if job_exception + payload['message'] = "#{message}: fail: #{payload['duration']} sec" + payload['job_status'] = 'fail' + payload['error_message'] = job_exception.message + payload['error'] = job_exception.class + payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace) + else + payload['message'] = "#{message}: done: #{payload['duration']} sec" + payload['job_status'] = 'done' + end + + convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) + + payload + end + + def parse_job(job) + job = job.dup + + # Add process id params + job['pid'] = ::Process.pid + + job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + + convert_to_iso8601(job, START_TIMESTAMP_FIELDS) + + job + end + + def convert_to_iso8601(payload, keys) + keys.each do |key| + payload[key] = format_time(payload[key]) if payload[key] + end + end + + def elapsed(start) + (current_time - start).round(3) + end + + def current_time + Gitlab::Metrics::System.monotonic_time + end + + def backtrace_cleaner + @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new + end + + def format_time(timestamp) + return timestamp if timestamp.is_a?(String) + + Time.at(timestamp).utc.iso8601(3) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb index c2b8d6de66e..b232ac4da33 100644 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -25,7 +25,7 @@ module Gitlab # can be only one shutdown thread in the process. def self.create_shutdown_thread mu_synchronize do - return unless @shutdown_thread.nil? + break unless @shutdown_thread.nil? @shutdown_thread = Thread.new { yield } end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index bb778f37096..c82320a6036 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,13 +1,15 @@ module Gitlab module SlashCommands class Command < BaseCommand - COMMANDS = [ - Gitlab::SlashCommands::IssueShow, - Gitlab::SlashCommands::IssueNew, - Gitlab::SlashCommands::IssueSearch, - Gitlab::SlashCommands::IssueMove, - Gitlab::SlashCommands::Deploy - ].freeze + def self.commands + [ + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::IssueMove, + Gitlab::SlashCommands::Deploy + ] + end def execute command, match = match_command @@ -37,7 +39,7 @@ module Gitlab private def available_commands - COMMANDS.select do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb new file mode 100644 index 00000000000..f357829ba3f --- /dev/null +++ b/lib/gitlab/sql/cte.rb @@ -0,0 +1,50 @@ +module Gitlab + module SQL + # Class for easily building CTE statements. + # + # Example: + # + # cte = CTE.new(:my_cte_name) + # ns = Arel::Table.new(:namespaces) + # + # cte << Namespace. + # where(ns[:parent_id].eq(some_namespace_id)) + # + # Namespace + # with(cte.to_arel). + # from(cte.alias_to(ns)) + class CTE + attr_reader :table, :query + + # name - The name of the CTE as a String or Symbol. + def initialize(name, query) + @table = Arel::Table.new(name) + @query = query + end + + # Returns the Arel relation for this CTE. + def to_arel + sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") + + Arel::Nodes::As.new(table, sql) + end + + # Returns an "AS" statement that aliases the CTE name as the given table + # name. This allows one to trick ActiveRecord into thinking it's selecting + # from an actual table, when in reality it's selecting from a CTE. + # + # alias_table - The Arel table to use as the alias. + def alias_to(alias_table) + Arel::Nodes::As.new(table, alias_table) + end + + # Applies the CTE to the given relation, returning a new one that will + # query from it. + def apply_to(relation) + relation.except(:where) + .with(to_arel) + .from(alias_to(relation.model.arel_table)) + end + end + end +end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 34bee6fecbe..922418966e9 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -128,17 +128,21 @@ module Gitlab end def all_repos - Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each_value do |repository_storage| + IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } + end end def user_home diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb new file mode 100644 index 00000000000..880e55f71df --- /dev/null +++ b/lib/gitlab/temporarily_allow.rb @@ -0,0 +1,42 @@ +module Gitlab + module TemporarilyAllow + TEMPORARILY_ALLOW_MUTEX = Mutex.new + + def temporarily_allow(key) + temporarily_allow_add(key, 1) + yield + ensure + temporarily_allow_add(key, -1) + end + + def temporarily_allowed?(key) + if RequestStore.active? + temporarily_allow_request_store[key] > 0 + else + TEMPORARILY_ALLOW_MUTEX.synchronize do + temporarily_allow_ivar[key] > 0 + end + end + end + + private + + def temporarily_allow_ivar + @temporarily_allow ||= Hash.new(0) + end + + def temporarily_allow_request_store + RequestStore[:temporarily_allow] ||= Hash.new(0) + end + + def temporarily_allow_add(key, value) + if RequestStore.active? + temporarily_allow_request_store[key] += value + else + TEMPORARILY_ALLOW_MUTEX.synchronize do + temporarily_allow_ivar[key] += value + end + end + end + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index d43eff5ba4a..694b01b272c 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -12,11 +12,16 @@ module Gitlab # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui_indigo'), - Theme.new(2, 'Dark', 'ui_dark'), - Theme.new(3, 'Light', 'ui_light'), - Theme.new(4, 'Blue', 'ui_blue'), - Theme.new(5, 'Green', 'ui_green') + Theme.new(1, 'Indigo', 'ui-indigo'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo'), + Theme.new(4, 'Blue', 'ui-blue'), + Theme.new(7, 'Light Blue', 'ui-light-blue'), + Theme.new(5, 'Green', 'ui-green'), + Theme.new(8, 'Light Green', 'ui-light-green'), + Theme.new(9, 'Red', 'ui-red'), + Theme.new(10, 'Light Red', 'ui-light-red'), + Theme.new(2, 'Dark', 'ui-dark'), + Theme.new(3, 'Light', 'ui-light') ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 7ce2e9d636e..dc2d91dfa23 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -9,9 +9,15 @@ module Gitlab # there is a strict limit on total execution time. See the RE2 documentation # at https://github.com/google/re2/wiki/Syntax for more details. class UntrustedRegexp - delegate :===, to: :regexp + require_dependency 're2' + + delegate :===, :source, to: :regexp + + def initialize(pattern, multiline: false) + if multiline + pattern = "(?m)#{pattern}" + end - def initialize(pattern) @regexp = RE2::Regexp.new(pattern, log_errors: false) raise RegexpError.new(regexp.error) unless regexp.ok? @@ -31,6 +37,41 @@ module Gitlab RE2.Replace(text, regexp, rewrite) end + def ==(other) + self.source == other.source + end + + # Handles regular expressions with the preferred RE2 library where possible + # via UntustedRegex. Falls back to Ruby's built-in regular expression library + # when the syntax would be invalid in RE2. + # + # One difference between these is `(?m)` multi-line mode. Ruby regex enables + # this by default, but also handles `^` and `$` differently. + # See: https://www.regular-expressions.info/modifiers.html + def self.with_fallback(pattern, multiline: false) + UntrustedRegexp.new(pattern, multiline: multiline) + rescue RegexpError + Regexp.new(pattern) + end + + def self.valid?(pattern) + !!self.fabricate(pattern) + rescue RegexpError + false + end + + def self.fabricate(pattern) + matches = pattern.match(%r{^/(?<regexp>.+)/(?<flags>[ismU]*)$}) + + raise RegexpError, 'Invalid regular expression!' if matches.nil? + + expression = matches[:regexp] + flags = matches[:flags] + expression.prepend("(?#{flags})") if flags.present? + + self.new(expression, multiline: false) + end + private attr_reader :regexp diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 0f9f939e204..38be75b7482 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -2,48 +2,91 @@ require 'resolv' module Gitlab class UrlBlocker - class << self - def blocked_url?(url, allow_private_networks: true, valid_ports: []) - return false if url.nil? + BlockedUrlError = Class.new(StandardError) - blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"] - blocked_ips.concat(Socket.ip_address_list.map(&:ip_address)) + class << self + def validate!(url, allow_localhost: false, allow_local_network: true, enforce_user: false, ports: [], protocols: []) + return true if url.nil? begin uri = Addressable::URI.parse(url) - # Allow imports from the GitLab instance itself but only from the configured ports - return false if internal?(uri) + rescue Addressable::URI::InvalidURIError + raise BlockedUrlError, "URI is invalid" + end - return true if blocked_port?(uri.port, valid_ports) - return true if blocked_user_or_hostname?(uri.user) - return true if blocked_user_or_hostname?(uri.hostname) + # Allow imports from the GitLab instance itself but only from the configured ports + return true if internal?(uri) - addrs_info = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM) - server_ips = addrs_info.map(&:ip_address) + port = uri.port || uri.default_port + validate_protocol!(uri.scheme, protocols) + validate_port!(port, ports) if ports.any? + validate_user!(uri.user) if enforce_user + validate_hostname!(uri.hostname) - return true if (blocked_ips & server_ips).any? - return true if !allow_private_networks && private_network?(addrs_info) - rescue Addressable::URI::InvalidURIError - return true + begin + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) rescue SocketError - return false + return true end + validate_localhost!(addrs_info) unless allow_localhost + validate_local_network!(addrs_info) unless allow_local_network + + true + end + + def blocked_url?(*args) + validate!(*args) + false + rescue BlockedUrlError + true end private - def blocked_port?(port, valid_ports) - return false if port.blank? || valid_ports.blank? + def validate_port!(port, ports) + return if port.blank? + # Only ports under 1024 are restricted + return if port >= 1024 + return if ports.include?(port) - port < 1024 && !valid_ports.include?(port) + raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" end - def blocked_user_or_hostname?(value) - return false if value.blank? + def validate_protocol!(protocol, protocols) + if protocol.blank? || (protocols.any? && !protocols.include?(protocol)) + raise BlockedUrlError, "Only allowed protocols are #{protocols.join(', ')}" + end + end - value !~ /\A\p{Alnum}/ + def validate_user!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Username needs to start with an alphanumeric character" + end + + def validate_hostname!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + end + + def validate_localhost!(addrs_info) + local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + return if (local_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to localhost are not allowed" + end + + def validate_local_network!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } + + raise BlockedUrlError, "Requests to the local network are not allowed" end def internal?(uri) @@ -60,10 +103,6 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end - def private_network?(addrs_info) - addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } - end - def config Gitlab.config end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 824e2d7251f..e64033b0dba 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -26,6 +26,8 @@ module Gitlab project_snippet_url(object.project, object) when Snippet snippet_url(object) + when Milestone + milestone_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 37d3512990e..dff0c97eeb4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -21,15 +21,16 @@ module Gitlab uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, + installation_type: Gitlab::INSTALLATION_TYPE, active_user_count: User.active.count, recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, edition: 'CE' } usage_data end + # rubocop:disable Metrics/AbcSize def system_usage_data { counts: { @@ -50,6 +51,12 @@ module Gitlab clusters: ::Clusters::Cluster.count, clusters_enabled: ::Clusters::Cluster.enabled.count, clusters_disabled: ::Clusters::Cluster.disabled.count, + clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, + clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, + clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, + clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, + clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, + clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, @@ -64,6 +71,7 @@ module Gitlab projects_imported_from_github: Project.where(import_type: 'github').count, protected_branches: ProtectedBranch.count, releases: Release.count, + remote_mirrors: RemoteMirror.count, snippets: Snippet.count, todos: Todo.count, uploads: Upload.count, @@ -82,13 +90,14 @@ module Gitlab def features_usage_data_ce { - signup: Gitlab::CurrentSettings.allow_signup?, - ldap: Gitlab.config.ldap.enabled, - gravatar: Gitlab::CurrentSettings.gravatar_enabled?, - omniauth: Gitlab.config.omniauth.enabled, - reply_by_email: Gitlab::IncomingEmail.enabled?, - container_registry: Gitlab.config.registry.enabled, - gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled + container_registry_enabled: Gitlab.config.registry.enabled, + gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, + gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, + ldap_enabled: Gitlab.config.ldap.enabled, + mattermost_enabled: Gitlab.config.mattermost.enabled, + omniauth_enabled: Gitlab.config.omniauth.enabled, + reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, + signup_enabled: Gitlab::CurrentSettings.allow_signup? } end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 24393f96d96..27560abfb96 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -51,7 +51,7 @@ module Gitlab return false unless can_access_git? if protected?(ProtectedBranch, project, ref) - user.can?(:delete_protected_branch, project) + user.can?(:push_to_delete_protected_branch, project) else user.can?(:push_code, project) end @@ -63,10 +63,12 @@ module Gitlab request_cache def can_push_to_branch?(ref) return false unless can_access_git? - return false unless user.can?(:push_code, project) || project.branch_allows_maintainer_push?(user, ref) + return false unless project + + return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref) if protected?(ProtectedBranch, project, ref) - project.user_can_push_to_empty_repo?(user) || protected_branch_accessible_to?(ref, action: :push) + protected_branch_accessible_to?(ref, action: :push) else true end @@ -101,6 +103,7 @@ module Gitlab def protected_branch_accessible_to?(ref, action:) ProtectedBranch.protected_ref_accessible_to?( ref, user, + project: project, action: action, protected_refs: project.protected_branches) end @@ -108,6 +111,7 @@ module Gitlab def protected_tag_accessible_to?(ref, action:) ProtectedTag.protected_ref_accessible_to?( ref, user, + project: project, action: action, protected_refs: project.protected_tags) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index dc9391f32cf..aeda66763e8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -27,6 +27,11 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + # Converts newlines into HTML line break elements + def nlbr(str) + ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe + end + def remove_line_breaks(str) str.gsub(/\r?\n/, '') end @@ -68,6 +73,10 @@ module Gitlab nil end + def bytes_to_megabytes(bytes) + bytes.to_f / Numeric::MEGABYTE + end + # Used in EE # Accepts either an Array or a String and returns an array def ensure_array_from_string(string_or_array) diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 8bf6bcb1fe2..7b2a62fed48 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -87,18 +87,28 @@ module Gitlab end def included(base = nil) - return super if base.nil? # Rails concern, ignoring it + super + + queue_verification(base) + end + alias_method :prepended, :included + + def extended(mod) super + queue_verification(mod.singleton_class) + end + + def queue_verification(base) + return unless ENV['STATIC_VERIFICATION'] + if base.is_a?(Class) # We could check for Class in `override` # This could be `nil` if `override` was never called Override.extensions[self]&.add_class(base) end end - alias_method :prepended, :included - def self.extensions @extensions ||= {} end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index 1ef369a4b67..167ba1b3149 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -7,13 +7,15 @@ module Gitlab @batch_size = batch_size @start = start @finish = finish + + fix_google_api_logger end # Yields a Range of IDs and a Hash of failed verifications (object => error) def run_batches(&blk) - relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches - range = relation.first.id..relation.last.id - failures = run_batch(relation) + all_relation.in_batches(of: batch_size, start: start, finish: finish) do |batch| # rubocop: disable Cop/InBatches + range = batch.first.id..batch.last.id + failures = run_batch_for(batch) yield(range, failures) end @@ -29,24 +31,56 @@ module Gitlab private - def run_batch(relation) - relation.map { |upload| verify(upload) }.compact.to_h + def run_batch_for(batch) + batch.map { |upload| verify(upload) }.compact.to_h end def verify(object) + local?(object) ? verify_local(object) : verify_remote(object) + rescue => err + failure(object, err.inspect) + end + + def verify_local(object) expected = expected_checksum(object) actual = actual_checksum(object) - raise 'Checksum missing' unless expected.present? - raise 'Checksum mismatch' unless expected == actual + return failure(object, 'Checksum missing') unless expected.present? + return failure(object, 'Checksum mismatch') unless expected == actual + + success + end + # We don't calculate checksum for remote objects, so just check existence + def verify_remote(object) + return failure(object, 'Remote object does not exist') unless remote_object_exists?(object) + + success + end + + def success nil - rescue => err - [object, err] + end + + def failure(object, message) + [object, message] + end + + # It's already set to Logger::INFO, but acts as if it is set to + # Logger::DEBUG, and this fixes it... + def fix_google_api_logger + if Object.const_defined?('Google::Apis') + Google::Apis.logger.level = Logger::INFO + end end # This should return an ActiveRecord::Relation suitable for calling #in_batches on - def relation + def all_relation + raise NotImplementedError.new + end + + # Should return true if the object is stored locally + def local?(_object) raise NotImplementedError.new end @@ -59,6 +93,11 @@ module Gitlab def actual_checksum(_object) raise NotImplementedError.new end + + # Be sure to perform a hard check of the remote object (don't just check DB value) + def remote_object_exists?(object) + raise NotImplementedError.new + end end end end diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb index 03500a61074..dbadfbde9e3 100644 --- a/lib/gitlab/verify/job_artifacts.rb +++ b/lib/gitlab/verify/job_artifacts.rb @@ -11,10 +11,14 @@ module Gitlab private - def relation + def all_relation ::Ci::JobArtifact.all end + def local?(artifact) + artifact.local_store? + end + def expected_checksum(artifact) artifact.file_sha256 end @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(artifact) Digest::SHA256.file(artifact.file.path).hexdigest end + + def remote_object_exists?(artifact) + artifact.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index fe51edbdeeb..d3f58a73ac7 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -11,10 +11,14 @@ module Gitlab private - def relation + def all_relation LfsObject.all end + def local?(lfs_object) + lfs_object.local_store? + end + def expected_checksum(lfs_object) lfs_object.oid end @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(lfs_object) LfsObject.calculate_oid(lfs_object.file.path) end + + def remote_object_exists?(lfs_object) + lfs_object.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb index dd138e6b92b..e190eaddc79 100644 --- a/lib/gitlab/verify/rake_task.rb +++ b/lib/gitlab/verify/rake_task.rb @@ -45,7 +45,7 @@ module Gitlab return unless verbose? failures.each do |object, error| - say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + say " - #{verifier.describe(object)}: #{error}".color(:red) end end end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 6972e517ea5..73fc43cb590 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -11,8 +11,12 @@ module Gitlab private - def relation - Upload.all + def all_relation + Upload.all.preload(:model) + end + + def local?(upload) + upload.local? end def expected_checksum(upload) @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(upload) Upload.hexdigest(upload.absolute_path) end + + def remote_object_exists?(upload) + upload.build_uploader.file.exists? + end end end end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 841fb681435..36162faa1eb 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -20,6 +20,10 @@ module Gitlab subject end + def present(**attributes) + self + end + class_methods do def presenter? true diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/webpack/dev_server_middleware.rb index 6aecf63231f..529f7d6a8d6 100644 --- a/lib/gitlab/middleware/webpack_proxy.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -3,8 +3,8 @@ # :nocov: module Gitlab - module Middleware - class WebpackProxy < Rack::Proxy + module Webpack + class DevServerMiddleware < Rack::Proxy def initialize(app = nil, opts = {}) @proxy_host = opts.fetch(:proxy_host, 'localhost') @proxy_port = opts.fetch(:proxy_port, 3808) @@ -15,6 +15,11 @@ module Gitlab def perform_request(env) if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + if relative_url_root = Rails.application.config.relative_url_root + env['SCRIPT_NAME'] = "" + env['REQUEST_PATH'].sub!(/\A#{Regexp.escape(relative_url_root)}/, '') + end + super(env) else @app.call(env) diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb new file mode 100644 index 00000000000..0c343e5bc1d --- /dev/null +++ b/lib/gitlab/webpack/manifest.rb @@ -0,0 +1,27 @@ +require 'webpack/rails/manifest' + +module Gitlab + module Webpack + class Manifest < ::Webpack::Rails::Manifest + # Raised if a supplied asset does not exist in the webpack manifest + AssetMissingError = Class.new(StandardError) + + class << self + def entrypoint_paths(source) + raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled? + + entrypoint = manifest["entrypoints"][source] + if entrypoint && entrypoint["assets"] + # Can be either a string or an array of strings. + # Do not include source maps as they are not javascript + [entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| + "/#{::Rails.configuration.webpack.public_path}/#{p}" + end + else + raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest" + end + end + end + end + end +end diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb new file mode 100644 index 00000000000..f97278f05cd --- /dev/null +++ b/lib/gitlab/wiki_file_finder.rb @@ -0,0 +1,23 @@ +module Gitlab + class WikiFileFinder < FileFinder + attr_reader :repository + + def initialize(project, ref) + @project = project + @ref = ref + @repository = project.wiki.repository + end + + private + + def search_filenames(query, except) + safe_query = Regexp.escape(query.tr(' ', '-')) + safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) + filenames = repository.ls_files(ref) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames.grep(safe_query).first(BATCH_SIZE) + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0b0d667d4fd..e893e46ee86 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -21,38 +21,23 @@ module Gitlab raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) project = repository.project - repo_path = repository.path_to_repo - params = { + + { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, - RepoPath: repo_path, - ShowAllRefs: show_all_refs - } - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - params[:GitalyServer] = server - - params - end - - def lfs_upload_ok(oid, size) - { - StoreLFSPath: LfsObjectUploader.workhorse_upload_path, - LfsOid: oid, - LfsSize: size + ShowAllRefs: show_all_refs, + Repository: repository.gitaly_repository.to_h, + RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitalyServer: { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } } end - def artifact_upload_ok - { TempPath: JobArtifactUploader.workhorse_upload_path } - end - def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'GetBlobRequest' => { @@ -74,18 +59,13 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:) + def send_git_archive(repository, ref:, format:, append_sha:) format ||= 'tar.gz' format.downcase! - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? - if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) - params.merge!( - 'GitalyServer' => gitaly_server_hash(repository), - 'GitalyRepository' => repository.gitaly_repository.to_h - ) - end + params['GitalyServer'] = gitaly_server_hash(repository) # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. params['DisableCache'] = true if git_archive_cache_disabled? @@ -96,8 +76,22 @@ module Gitlab ] end + def send_git_snapshot(repository) + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new( + repository: repository.gitaly_repository + ).to_json + } + + [ + SEND_DATA_HEADER, + "git-snapshot:#{encode(params)}" + ] + end + def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawDiffRequest' => Gitaly::RawDiffRequest.new( @@ -115,7 +109,7 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawPatchRequest' => Gitaly::RawPatchRequest.new( diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index f30dd995695..36859b4d025 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,3 +1,4 @@ +require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' @@ -42,22 +43,6 @@ module GoogleApi true end - def projects_list - service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new - service.authorization = access_token - - service.fetch_all(items: :projects) do |token| - service.list_projects(page_token: token, options: user_agent_header) - end - end - - def projects_get_billing_info(project_id) - service = Google::Apis::CloudbillingV1::CloudbillingService.new - service.authorization = access_token - - service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) - end - def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb index 33e450d7f0a..704813dfdf0 100644 --- a/lib/mattermost/command.rb +++ b/lib/mattermost/command.rb @@ -1,7 +1,7 @@ module Mattermost class Command < Client def create(params) - response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create", + response = session_post('/api/v4/commands', body: params.to_json) response['token'] diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 85f78e44f32..2aa7a2f64d8 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -112,7 +112,7 @@ module Mattermost end def destroy - post('/api/v3/users/logout') + post('/api/v4/users/logout') end def oauth_uri @@ -120,7 +120,7 @@ module Mattermost @oauth_uri = nil - response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + response = get('/oauth/gitlab/login', follow_redirects: false, format: 'text/html') return unless (300...400) === response.code redirect_uri = response.headers['location'] diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 75513a9ba04..95c2f6f9d6b 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,14 +1,14 @@ module Mattermost class Team < Client - # Returns **all** teams for an admin + # Returns all teams that the current user is a member of def all - session_get('/api/v3/teams/all').values + session_get("/api/v4/users/me/teams") end # Creates a team on the linked Mattermost instance, the team admin will be the # `current_user` passed to the Mattermost::Client instance def create(name:, display_name:, type:) - session_post('/api/v3/teams/create', body: { + session_post('/api/v4/teams', body: { name: name, display_name: display_name, type: type diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index c08d3e933a8..226ee1373db 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -30,7 +30,7 @@ module MicrosoftTeams result = { 'sections' => [] } result['title'] = options[:title] - result['summary'] = options[:pretext] + result['summary'] = options[:summary] result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare attachments = options[:attachments] diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb new file mode 100644 index 00000000000..64634f789da --- /dev/null +++ b/lib/mysql_zero_date.rb @@ -0,0 +1,18 @@ +# Disable NO_ZERO_DATE mode for mysql in rails 5. +# We use zero date as a default value +# (config/initializers/active_record_mysql_timestamp.rb), in +# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) +# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode +# is not sufficient. + +require 'active_record/connection_adapters/abstract_mysql_adapter' + +module MysqlZeroDate + def configure_connection + super + + @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end + +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) if Gitlab.rails5? diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb new file mode 100644 index 00000000000..61a69e7ffe4 --- /dev/null +++ b/lib/object_storage/direct_upload.rb @@ -0,0 +1,166 @@ +module ObjectStorage + # + # The DirectUpload c;ass generates a set of presigned URLs + # that can be used to upload data to object storage from untrusted component: Workhorse, Runner? + # + # For Google it assumes that the platform supports variable Content-Length. + # + # For AWS it initiates Multipart Upload and presignes a set of part uploads. + # Class calculates the best part size to be able to upload up to asked maximum size. + # The number of generated parts will never go above 100, + # but we will always try to reduce amount of generated parts. + # The part size is rounded-up to 5MB. + # + class DirectUpload + include Gitlab::Utils::StrongMemoize + + TIMEOUT = 4.hours + EXPIRE_OFFSET = 15.minutes + + MAXIMUM_MULTIPART_PARTS = 100 + MINIMUM_MULTIPART_SIZE = 5.megabytes + + attr_reader :credentials, :bucket_name, :object_name + attr_reader :has_length, :maximum_size + + def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil) + unless has_length + raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size + end + + @credentials = credentials + @bucket_name = bucket_name + @object_name = object_name + @has_length = has_length + @maximum_size = maximum_size + end + + def to_hash + { + Timeout: TIMEOUT, + GetURL: get_url, + StoreURL: store_url, + DeleteURL: delete_url, + MultipartUpload: multipart_upload_hash + }.compact + end + + def multipart_upload_hash + return unless requires_multipart_upload? + + { + PartSize: rounded_multipart_part_size, + PartURLs: multipart_part_urls, + CompleteURL: multipart_complete_url, + AbortURL: multipart_abort_url + } + end + + def provider + credentials[:provider].to_s + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + def get_url + connection.get_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html + def delete_url + connection.delete_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html + def store_url + connection.put_object_url(bucket_name, object_name, expire_at, upload_options) + end + + def multipart_part_urls + Array.new(number_of_multipart_parts) do |part_index| + multipart_part_upload_url(part_index + 1) + end + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + def multipart_part_upload_url(part_number) + connection.signed_url({ + method: 'PUT', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id, partNumber: part_number }, + headers: upload_options + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html + def multipart_complete_url + connection.signed_url({ + method: 'POST', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id }, + headers: { 'Content-Type' => 'application/xml' } + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html + def multipart_abort_url + connection.signed_url({ + method: 'DELETE', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id } + }, expire_at) + end + + private + + def rounded_multipart_part_size + # round multipart_part_size up to minimum_mulitpart_size + (multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE + end + + def multipart_part_size + maximum_size / number_of_multipart_parts + end + + def number_of_multipart_parts + [ + # round maximum_size up to minimum_mulitpart_size + (maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE, + MAXIMUM_MULTIPART_PARTS + ].min + end + + def aws? + provider == 'AWS' + end + + def requires_multipart_upload? + aws? && !has_length + end + + def upload_id + return unless requires_multipart_upload? + + strong_memoize(:upload_id) do + new_upload = connection.initiate_multipart_upload(bucket_name, object_name) + new_upload.body["UploadId"] + end + end + + def expire_at + strong_memoize(:expire_at) do + Time.now + TIMEOUT + EXPIRE_OFFSET + end + end + + def upload_options + { 'Content-Type' => 'application/octet-stream' } + end + + def connection + @connection ||= ::Fog::Storage.new(credentials) + end + end +end diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb new file mode 100644 index 00000000000..ebdb5c7faf0 --- /dev/null +++ b/lib/omni_auth/strategies/jwt.rb @@ -0,0 +1,60 @@ +require 'omniauth' +require 'jwt' + +module OmniAuth + module Strategies + class Jwt + ClaimInvalid = Class.new(StandardError) + + include OmniAuth::Strategy + + args [:secret] + + option :secret, nil + option :algorithm, 'HS256' + option :uid_claim, 'email' + option :required_claims, %w(name email) + option :info_map, { name: "name", email: "email" } + option :auth_url, nil + option :valid_within, nil + + uid { decoded[options.uid_claim] } + + extra do + { raw_info: decoded } + end + + info do + options.info_map.each_with_object({}) do |(k, v), h| + h[k.to_s] = decoded[v.to_s] + end + end + + def request_phase + redirect options.auth_url + end + + def decoded + @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first + + (options.required_claims || []).each do |field| + raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s) + end + + raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"] + + if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within + raise ClaimInvalid, "'iat' timestamp claim is too skewed from present" + end + + @decoded + end + + def callback_phase + super + rescue ClaimInvalid => e + fail! :claim_invalid, e + end + end + end +end diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb index 7cfe76b7b71..9beb442bfa3 100644 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ b/lib/peek/rblineprof/custom_controller_helpers.rb @@ -41,10 +41,10 @@ module Peek ] end.sort_by{ |a,b,c,d,e,f| -f } - output = "<div class='modal-dialog modal-full'><div class='modal-content'>" + output = "<div class='modal-dialog modal-xl'><div class='modal-content'>" output << "<div class='modal-header'>" - output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>" output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" + output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>×</span></button>" output << "</div>" output << "<div class='modal-body'>" diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb index a17ae55910e..06e96f969f1 100644 --- a/lib/rspec_flaky/config.rb +++ b/lib/rspec_flaky/config.rb @@ -1,9 +1,7 @@ -require 'json' - module RspecFlaky class Config def self.generate_report? - ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) end def self.suite_flaky_examples_report_path diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb index 973c95b0212..dea23c325be 100644 --- a/lib/rspec_flaky/flaky_examples_collection.rb +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -1,11 +1,9 @@ -require 'json' +require 'active_support/hash_with_indifferent_access' + +require_relative 'flaky_example' module RspecFlaky class FlakyExamplesCollection < SimpleDelegator - def self.from_json(json) - new(JSON.parse(json)) - end - def initialize(collection = {}) unless collection.is_a?(Hash) raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" @@ -22,7 +20,7 @@ module RspecFlaky super(Hash[collection_of_flaky_examples]) end - def to_report + def to_h Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 4a5bfec9967..9cd0c38cb55 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -1,5 +1,11 @@ require 'json' +require_dependency 'rspec_flaky/config' +require_dependency 'rspec_flaky/example' +require_dependency 'rspec_flaky/flaky_example' +require_dependency 'rspec_flaky/flaky_examples_collection' +require_dependency 'rspec_flaky/report' + module RspecFlaky class Listener # - suite_flaky_examples: contains all the currently tracked flacky example @@ -9,7 +15,7 @@ module RspecFlaky attr_reader :suite_flaky_examples, :flaky_examples def initialize(suite_flaky_examples_json = nil) - @flaky_examples = FlakyExamplesCollection.new + @flaky_examples = RspecFlaky::FlakyExamplesCollection.new @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end @@ -18,47 +24,36 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example) } flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) + # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_h) - write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) + RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path) + # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end - def to_report(examples) - Hash[examples.map { |k, ex| [k, ex.to_h] }] - end - private def init_suite_flaky_examples(suite_flaky_examples_json = nil) - unless suite_flaky_examples_json + if suite_flaky_examples_json + RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples + else return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples end - - FlakyExamplesCollection.from_json(suite_flaky_examples_json) - end - - def write_report_file(examples_collection, file_path) - return unless RspecFlaky::Config.generate_report? - - report_path_dir = File.dirname(file_path) - FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - - File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb new file mode 100644 index 00000000000..1c362fdd20d --- /dev/null +++ b/lib/rspec_flaky/report.rb @@ -0,0 +1,54 @@ +require 'json' +require 'time' + +require_dependency 'rspec_flaky/config' +require_dependency 'rspec_flaky/flaky_examples_collection' + +module RspecFlaky + # This class is responsible for loading/saving JSON reports, and pruning + # outdated examples. + class Report < SimpleDelegator + OUTDATED_DAYS_THRESHOLD = 90 + + attr_reader :flaky_examples + + def self.load(file_path) + load_json(File.read(file_path)) + end + + def self.load_json(json) + new(RspecFlaky::FlakyExamplesCollection.new(JSON.parse(json))) + end + + def initialize(flaky_examples) + unless flaky_examples.is_a?(RspecFlaky::FlakyExamplesCollection) + raise ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" + end + + @flaky_examples = flaky_examples + super(flaky_examples) + end + + def write(file_path) + unless RspecFlaky::Config.generate_report? + puts "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" # rubocop:disable Rails/Output + return + end + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) + + File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) + end + + def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) + outdated_date_threshold = Time.now - (3600 * 24 * days) + updated_hash = flaky_examples.dup + .delete_if do |uid, hash| + hash[:last_flaky_at] && Time.parse(hash[:last_flaky_at]).to_i < outdated_date_threshold.to_i + end + + self.class.new(RspecFlaky::FlakyExamplesCollection.new(updated_hash)) + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 0e27a28ea6e..72eb8adcce2 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -31,27 +31,27 @@ map $http_upgrade $connection_upgrade_gitlab { log_format gitlab_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent"; ## Remove private_token from the request URI -# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&feed_token=unfiltered&... map $request_uri $gitlab_temp_request_uri_1 { default $request_uri; ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } ## Remove authenticity_token from the request URI -# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=unfiltered&... map $gitlab_temp_request_uri_1 $gitlab_temp_request_uri_2 { default $gitlab_temp_request_uri_1; ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } -## Remove rss_token from the request URI -# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +## Remove feed_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=[FILTERED]&... map $gitlab_temp_request_uri_2 $gitlab_filtered_request_uri { default $gitlab_temp_request_uri_2; - ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; + ~(?i)^(?<start>.*)(?<temp>[\?&]feed[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } ## A version of the referer without the query string diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 8218d68f9ba..2e3799d5e1b 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -36,27 +36,27 @@ map $http_upgrade $connection_upgrade_gitlab_ssl { log_format gitlab_ssl_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent"; ## Remove private_token from the request URI -# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&feed_token=unfiltered&... map $request_uri $gitlab_ssl_temp_request_uri_1 { default $request_uri; ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } ## Remove authenticity_token from the request URI -# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=unfiltered&... map $gitlab_ssl_temp_request_uri_1 $gitlab_ssl_temp_request_uri_2 { default $gitlab_ssl_temp_request_uri_1; ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } -## Remove rss_token from the request URI -# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... -# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +## Remove feed_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&feed_token=[FILTERED]&... map $gitlab_ssl_temp_request_uri_2 $gitlab_ssl_filtered_request_uri { default $gitlab_ssl_temp_request_uri_2; - ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; + ~(?i)^(?<start>.*)(?<temp>[\?&]feed[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; } ## A version of the referer without the query string diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb index b8446300f72..09b57c7b408 100644 --- a/lib/system_check/orphans/namespace_check.rb +++ b/lib/system_check/orphans/namespace_check.rb @@ -4,13 +4,15 @@ module SystemCheck set_name 'Orphaned namespaces:' def multi_check - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) - - orphans = (toplevel_namespace_dirs - existing_namespaces) - print_orphans(orphans, storage_name) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end end clear_namespaces! # releases memory when check finishes diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb index 9b6b2429783..2695c658874 100644 --- a/lib/system_check/orphans/repository_check.rb +++ b/lib/system_check/orphans/repository_check.rb @@ -5,14 +5,18 @@ module SystemCheck attr_accessor :orphans def multi_check - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + storage_path = repository_storage.legacy_disk_path - repositories = disk_repositories(repository_storage['path']) - orphans = (repositories - fetch_repositories(storage_name)) + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) - print_orphans(orphans, storage_name) + repositories = disk_repositories(storage_path) + orphans = (repositories - fetch_repositories(storage_name)) + + print_orphans(orphans, storage_name) + end end end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index d268f501b4a..99c9e984107 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -43,7 +43,7 @@ module SystemCheck # # @param [SystemCheck::BaseCheck] check_klass def run_check(check_klass) - $stdout.print "#{check_klass.display_name} ... " + print_display_name(check_klass) check = check_klass.new @@ -60,18 +60,18 @@ module SystemCheck end if check.check? - $stdout.puts check_klass.check_pass.color(:green) + print_check_pass(check_klass) else - $stdout.puts check_klass.check_fail.color(:red) + print_check_failure(check_klass) if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' if check.repair! - $stdout.puts 'Success'.color(:green) + print_success return else - $stdout.puts 'Failed'.color(:red) + print_failure end end @@ -83,6 +83,26 @@ module SystemCheck private + def print_display_name(check_klass) + $stdout.print "#{check_klass.display_name} ... " + end + + def print_check_pass(check_klass) + $stdout.puts check_klass.check_pass.color(:green) + end + + def print_check_failure(check_klass) + $stdout.puts check_klass.check_fail.color(:red) + end + + def print_success + $stdout.puts 'Success'.color(:green) + end + + def print_failure + $stdout.puts 'Failed'.color(:red) + end + # Prints header content for the series of checks to be executed for this component # # @param [String] component name of the component relative to the checks being executed diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 564aa141952..cb4d5abffbc 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -6,17 +6,22 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do Gitlab::Redis::Cache.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", - count: REDIS_CLEAR_BATCH_SIZE - ) + cache_key_pattern = %W[#{Gitlab::Redis::Cache::CACHE_NAMESPACE}* + projects/*/pipeline_status] - redis.del(*keys) if keys.any? + cache_key_pattern.each do |match| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: match, + count: REDIS_CLEAR_BATCH_SIZE + ) - break if cursor == REDIS_SCAN_START_STOP + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 4b4881cecb8..4bec013a141 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` + output = `bundle exec flay --mass 35 app/ lib/gitlab/ ee/ 2> #{File::NULL}` if output.include?("Similar code found") || output.include?("IDENTICAL code found") puts output diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 247d7be7d78..21998dd2f5b 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -4,7 +4,7 @@ namespace :gettext do # Customize list of translatable files # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate - folders = %W(app lib config #{locale_path}).join(',') + folders = %W(ee app lib config #{locale_path}).join(',') exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( @@ -16,7 +16,6 @@ namespace :gettext do # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) - Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end @@ -50,6 +49,41 @@ namespace :gettext do end end + task :updated_check do + # Removing all pre-translated files speeds up `gettext:find` as the + # files don't need to be merged. + # Having `LC_MESSAGES/gitlab.mo files present also confuses the output. + FileUtils.rm Dir['locale/**/gitlab.*'] + + # Make sure we start out with a clean pot.file + `git checkout -- locale/gitlab.pot` + + # `gettext:find` writes touches to temp files to `stderr` which would cause + # `static-analysis` to report failures. We can ignore these. + silence_stream($stderr) do + Rake::Task['gettext:find'].invoke + end + + pot_diff = `git diff -- locale/gitlab.pot`.strip + + # reset the locale folder for potential next tasks + `git checkout -- locale` + + if pot_diff.present? + raise <<~MSG + Newly translated strings found, please add them to `gitlab.pot` by running: + + rm locale/**/gitlab.*; bin/rake gettext:find; git checkout -- locale/*/gitlab.po + + Then commit and push the resulting changes to `locale/gitlab.pot`. + + The diff was: + + #{pot_diff} + MSG + end + end + def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake new file mode 100644 index 00000000000..bfca4bfb3f7 --- /dev/null +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -0,0 +1,25 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Migrate files for artifacts to comply with new storage format" +namespace :gitlab do + namespace :artifacts do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts') + + Ci::Build.joins(:project) + .with_artifacts_stored_locally + .find_each(batch_size: 10) do |build| + begin + build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) + build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + + logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + rescue => e + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 24e37f6c6cc..e96fbb64372 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -6,7 +6,6 @@ namespace :gitlab do desc "GitLab | Create a backup of the GitLab system" task create: :gitlab_environment do warn_user_is_not_gitlab - configure_cron_mode Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke @@ -17,7 +16,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:lfs:create"].invoke Rake::Task["gitlab:backup:registry:create"].invoke - backup = Backup::Manager.new + backup = Backup::Manager.new(progress) backup.pack backup.cleanup backup.remove_old @@ -27,9 +26,8 @@ namespace :gitlab do desc 'GitLab | Restore a previously created backup' task restore: :gitlab_environment do warn_user_is_not_gitlab - configure_cron_mode - backup = Backup::Manager.new + backup = Backup::Manager.new(progress) backup.unpack unless backup.skipped?('db') @@ -49,9 +47,9 @@ namespace :gitlab do # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade - $progress.puts 'Cleaning the database ... '.color(:blue) + progress.puts 'Cleaning the database ... '.color(:blue) Rake::Task['gitlab:db:drop_tables'].invoke - $progress.puts 'done'.color(:green) + progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) @@ -74,173 +72,173 @@ namespace :gitlab do namespace :repo do task create: :gitlab_environment do - $progress.puts "Dumping repositories ...".color(:blue) + progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Repository.new.dump - $progress.puts "done".color(:green) + Backup::Repository.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring repositories ...".color(:blue) - Backup::Repository.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring repositories ...".color(:blue) + Backup::Repository.new(progress).restore + progress.puts "done".color(:green) end end namespace :db do task create: :gitlab_environment do - $progress.puts "Dumping database ... ".color(:blue) + progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Database.new.dump - $progress.puts "done".color(:green) + Backup::Database.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring database ... ".color(:blue) - Backup::Database.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring database ... ".color(:blue) + Backup::Database.new(progress).restore + progress.puts "done".color(:green) end end namespace :builds do task create: :gitlab_environment do - $progress.puts "Dumping builds ... ".color(:blue) + progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Builds.new.dump - $progress.puts "done".color(:green) + Backup::Builds.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring builds ... ".color(:blue) - Backup::Builds.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring builds ... ".color(:blue) + Backup::Builds.new(progress).restore + progress.puts "done".color(:green) end end namespace :uploads do task create: :gitlab_environment do - $progress.puts "Dumping uploads ... ".color(:blue) + progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Uploads.new.dump - $progress.puts "done".color(:green) + Backup::Uploads.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring uploads ... ".color(:blue) - Backup::Uploads.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring uploads ... ".color(:blue) + Backup::Uploads.new(progress).restore + progress.puts "done".color(:green) end end namespace :artifacts do task create: :gitlab_environment do - $progress.puts "Dumping artifacts ... ".color(:blue) + progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Artifacts.new.dump - $progress.puts "done".color(:green) + Backup::Artifacts.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring artifacts ... ".color(:blue) - Backup::Artifacts.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring artifacts ... ".color(:blue) + Backup::Artifacts.new(progress).restore + progress.puts "done".color(:green) end end namespace :pages do task create: :gitlab_environment do - $progress.puts "Dumping pages ... ".color(:blue) + progress.puts "Dumping pages ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("pages") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Pages.new.dump - $progress.puts "done".color(:green) + Backup::Pages.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring pages ... ".color(:blue) - Backup::Pages.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring pages ... ".color(:blue) + Backup::Pages.new(progress).restore + progress.puts "done".color(:green) end end namespace :lfs do task create: :gitlab_environment do - $progress.puts "Dumping lfs objects ... ".color(:blue) + progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Lfs.new.dump - $progress.puts "done".color(:green) + Backup::Lfs.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring lfs objects ... ".color(:blue) - Backup::Lfs.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring lfs objects ... ".color(:blue) + Backup::Lfs.new(progress).restore + progress.puts "done".color(:green) end end namespace :registry do task create: :gitlab_environment do - $progress.puts "Dumping container registry images ... ".color(:blue) + progress.puts "Dumping container registry images ... ".color(:blue) if Gitlab.config.registry.enabled if ENV["SKIP"] && ENV["SKIP"].include?("registry") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Registry.new.dump - $progress.puts "done".color(:green) + Backup::Registry.new(progress).dump + progress.puts "done".color(:green) end else - $progress.puts "[DISABLED]".color(:cyan) + progress.puts "[DISABLED]".color(:cyan) end end task restore: :gitlab_environment do - $progress.puts "Restoring container registry images ... ".color(:blue) + progress.puts "Restoring container registry images ... ".color(:blue) if Gitlab.config.registry.enabled - Backup::Registry.new.restore - $progress.puts "done".color(:green) + Backup::Registry.new(progress).restore + progress.puts "done".color(:green) else - $progress.puts "[DISABLED]".color(:cyan) + progress.puts "[DISABLED]".color(:cyan) end end end - def configure_cron_mode + def progress if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a # StringIO. require 'stringio' - $progress = StringIO.new + StringIO.new else - $progress = $stdout + $stdout end end end # namespace end: backup diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2403f57f05a..a8acafa9cd9 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -44,11 +44,13 @@ namespace :gitlab do start_checking "GitLab Shell" check_gitlab_shell - check_repo_base_exists - check_repo_base_is_not_symlink - check_repo_base_user_and_group - check_repo_base_permissions - check_repos_hooks_directory_is_link + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + check_repo_base_exists + check_repo_base_is_not_symlink + check_repo_base_user_and_group + check_repo_base_permissions + check_repos_hooks_directory_is_link + end check_gitlab_shell_self_test finished_checking "GitLab Shell" @@ -61,7 +63,7 @@ namespace :gitlab do puts "Repo base directory exists?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " if File.exist?(repo_base_path) @@ -86,7 +88,7 @@ namespace :gitlab do puts "Repo storage directories are symlinks?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -110,7 +112,7 @@ namespace :gitlab do puts "Repo paths access is drwxrws---?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -140,7 +142,7 @@ namespace :gitlab do puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -427,10 +429,7 @@ namespace :gitlab do user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| - File.join( - p.repository_storage_path, - "#{p.disk_path}.git" - ) + p.repository.path_to_repo end repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 2453079911d..52ae1330d7f 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -12,7 +12,7 @@ namespace :gitlab do namespaces = Namespace.pluck(:path) namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = repository_storage['path'] + git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } all_dirs = Dir.glob(git_base_path + '/*') puts git_base_path.color(:yellow) @@ -54,7 +54,8 @@ namespace :gitlab do move_suffix = "+orphaned+#{Time.now.to_i}" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = repository_storage['path'] + repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } + # Look for global repos (legacy, depth 1) and normal repos (depth 2) IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 45e9a1a1c72..6de739e9515 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -47,7 +47,7 @@ namespace :gitlab do puts "" puts "GitLab information".color(:yellow) puts "Version:\t#{Gitlab::VERSION}" - puts "Revision:\t#{Gitlab::REVISION}" + puts "Revision:\t#{Gitlab.revision}" puts "Directory:\t#{Rails.root}" puts "DB Adapter:\t#{database_adapter}" puts "URL:\t\t#{Gitlab.config.gitlab.url}" @@ -67,8 +67,10 @@ namespace :gitlab do puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" - Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage['path']}" + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage.legacy_disk_path}" + end end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake new file mode 100644 index 00000000000..a45e5ca91e0 --- /dev/null +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -0,0 +1,22 @@ +require 'logger' + +desc "GitLab | Migrate LFS objects to remote storage" +namespace :gitlab do + namespace :lfs do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to object storage') + + LfsObject.with_files_stored_locally + .find_each(batch_size: 10) do |lfs_object| + begin + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index d7f28691098..b854c34a8e5 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -10,9 +10,8 @@ namespace :gitlab do end scope.find_each do |project| - base = File.join(project.repository_storage_path, project.disk_path) - puts base + '.git' - puts base + '.wiki.git' + puts project.repository.path_to_repo + puts project.wiki.repository.path_to_repo end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake new file mode 100644 index 00000000000..100e480bd66 --- /dev/null +++ b/lib/tasks/gitlab/pages.rake @@ -0,0 +1,9 @@ +namespace :gitlab do + namespace :pages do + desc 'Ping the pages admin API' + task admin_ping: :gitlab_environment do + Gitlab::PagesClient.ping + puts "OK: gitlab-pages admin API is reachable" + end + end +end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 1d903c81358..f71e69987cb 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -1,9 +1,20 @@ namespace :gitlab do desc "GitLab | Setup production application" task setup: :gitlab_environment do + check_gitaly_connection setup_db end + def check_gitaly_connection + Gitlab.config.repositories.storages.each do |name, _details| + Gitlab::GitalyClient::ServerService.new(name).info + end + rescue GRPC::Unavailable => ex + puts "Failed to connect to Gitaly...".color(:red) + puts "Error: #{ex}" + exit 1 + end + def setup_db warn_user_is_not_gitlab diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 8ac73bc8ff2..f539b1df955 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,6 +2,24 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do + storage_migrator = Gitlab::HashedStorage::Migrator.new + helper = Gitlab::HashedStorage::RakeHelper + + if helper.range_single_item? + project = Project.with_unmigrated_storage.find_by(id: helper.range_from) + + unless project + puts "There are no projects requiring storage migration with ID=#{helper.range_from}" + + next + end + + puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." + storage_migrator.migrate(project) + + next + end + legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 @@ -10,10 +28,10 @@ namespace :gitlab do next end - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - project_id_batches do |start, finish| - StorageMigratorWorker.perform_async(start, finish) + helper.project_id_batches do |start, finish| + storage_migrator.bulk_schedule(start, finish) print '.' end @@ -23,118 +41,50 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' task legacy_projects: :environment do - relation_summary('projects', Project.without_storage_feature(:repository)) + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Legacy Storage' task list_legacy_projects: :environment do - projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) + helper = Gitlab::HashedStorage::RakeHelper + helper.projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' task hashed_projects: :environment do - relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Hashed Storage' task list_hashed_projects: :environment do - projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) + helper = Gitlab::HashedStorage::RakeHelper + helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) end desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' task legacy_attachments: :environment do - relation_summary('attachments using Legacy Storage', legacy_attachments_relation) + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation) end desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' task list_legacy_attachments: :environment do - attachments_list('attachments using Legacy Storage', legacy_attachments_relation) + helper = Gitlab::HashedStorage::RakeHelper + helper.attachments_list('attachments using Legacy Storage', helper.legacy_attachments_relation) end desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage' task hashed_attachments: :environment do - relation_summary('attachments using Hashed Storage', hashed_attachments_relation) + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('attachments using Hashed Storage', helper.hashed_attachments_relation) end desc 'Gitlab | Storage | List existing project attachments using Hashed Storage' task list_hashed_attachments: :environment do - attachments_list('attachments using Hashed Storage', hashed_attachments_relation) - end - - def batch_size - ENV.fetch('BATCH', 200).to_i - end - - def project_id_batches(&block) - Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches - ids = relation.pluck(:id) - - yield ids.min, ids.max - end - end - - def legacy_attachments_relation - Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) - JOIN projects - ON (uploads.model_type='Project' AND uploads.model_id=projects.id) - SQL - end - - def hashed_attachments_relation - Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) - JOIN projects - ON (uploads.model_type='Project' AND uploads.model_id=projects.id) - SQL - end - - def relation_summary(relation_name, relation) - relation_count = relation.count - puts "* Found #{relation_count} #{relation_name}".color(:green) - - relation_count - end - - def projects_list(relation_name, relation) - relation_count = relation_summary(relation_name, relation) - - projects = relation.with_route - limit = ENV.fetch('LIMIT', 500).to_i - - return unless relation_count > 0 - - puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit - - counter = 0 - projects.find_in_batches(batch_size: batch_size) do |batch| - batch.each do |project| - counter += 1 - - puts " - #{project.full_path} (id: #{project.id})".color(:red) - - return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator - end - end - end - - def attachments_list(relation_name, relation) - relation_count = relation_summary(relation_name, relation) - - limit = ENV.fetch('LIMIT', 500).to_i - - return unless relation_count > 0 - - puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit - - counter = 0 - relation.find_in_batches(batch_size: batch_size) do |batch| - batch.each do |upload| - counter += 1 - - puts " - #{upload.path} (id: #{upload.id})".color(:red) - - return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator - end - end + helper = Gitlab::HashedStorage::RakeHelper + helper.attachments_list('attachments using Hashed Storage', helper.hashed_attachments_relation) end end end diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index 523b0fa055b..2222807fe13 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -4,7 +4,6 @@ namespace :gitlab do cmds = [ %w(rake brakeman), %w(rake rubocop), - %w(rake spinach), %w(rake spec), %w(rake karma) ] diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake index fd2a4f2d11a..ddcca69711f 100644 --- a/lib/tasks/gitlab/traces.rake +++ b/lib/tasks/gitlab/traces.rake @@ -8,9 +8,7 @@ namespace :gitlab do logger = Logger.new(STDOUT) logger.info('Archiving legacy traces') - Ci::Build.finished - .where('NOT EXISTS (?)', - Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + Ci::Build.finished.without_archived_trace .order(id: :asc) .find_in_batches(batch_size: 1000) do |jobs| job_ids = jobs.map { |job| [job.id] } diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 7728c485e8d..6b22499a5c8 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :two_factor do desc "GitLab | Disable Two-factor authentication (2FA) for all users" - task disable_for_all_users: :environment do + task disable_for_all_users: :gitlab_environment do scope = User.with_two_factor count = scope.count diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake new file mode 100644 index 00000000000..78e18992a8e --- /dev/null +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -0,0 +1,34 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| + batch_size = ENV.fetch('BATCH', 200).to_i + @to_store = ObjectStorage::Store::REMOTE + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + end + + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, + @mounted_as, + @to_store) + puts "Enqueued job ##{index}: #{job}" + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + end + + def uploads + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where(store: [nil, ObjectStorage::Store::LOCAL], + uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) + end + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index aafbe52e5f8..fc59b3f937d 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -9,7 +9,10 @@ class GithubImport def initialize(token, gitlab_username, project_path, extras) @options = { token: token } @project_path = project_path - @current_user = User.find_by_username(gitlab_username) + @current_user = User.find_by(username: gitlab_username) + + raise "GitLab user #{gitlab_username} not found. Please specify a valid username." unless @current_user + @github_repo = extras.empty? ? nil : extras.first end @@ -50,7 +53,7 @@ class GithubImport end if import_success - @project.import_finish + @project.after_import puts "Import finished. Timings: #{timings}".color(:green) else puts "Import was not successful. Errors were as follows:" diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index fe5032cae18..006fcdd31a4 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -17,45 +17,54 @@ unless Rails.env.production? Rake::Task['eslint'].invoke end + desc "GitLab | lint | Lint HAML files" + task :haml do + begin + Rake::Task['haml_lint'].invoke + rescue RuntimeError # The haml_lint tasks raise a RuntimeError + exit(1) + end + end + desc "GitLab | lint | Run several lint checks" task :all do status = 0 %w[ config_lint - haml_lint + lint:haml scss_lint flay gettext:lint + gettext:updated_check lint:static_verification ].each do |task| pid = Process.fork do - rd, wr = IO.pipe + rd_out, wr_out = IO.pipe + rd_err, wr_err = IO.pipe stdout = $stdout.dup stderr = $stderr.dup - $stdout.reopen(wr) - $stderr.reopen(wr) + $stdout.reopen(wr_out) + $stderr.reopen(wr_err) begin - begin - Rake::Task[task].invoke - rescue RuntimeError # The haml_lint tasks raise a RuntimeError - exit(1) - end + Rake::Task[task].invoke rescue SystemExit => ex - msg = "*** Rake task #{task} failed with the following error(s):" + msg = "*** Rake task #{task} exited:" + raise ex + rescue => ex + msg = "*** Rake task #{task} raised #{ex.class}:" raise ex ensure $stdout.reopen(stdout) $stderr.reopen(stderr) - wr.close + wr_out.close + wr_err.close + + warn "\n#{msg}\n\n" if msg - if msg - warn "\n#{msg}\n\n" - IO.copy_stream(rd, $stderr) - else - IO.copy_stream(rd, $stdout) - end + IO.copy_stream(rd_out, $stdout) + IO.copy_stream(rd_err, $stderr) end end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 151f42a2222..9b05876034c 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,6 +1,8 @@ require Rails.root.join('db/migrate/limits_to_mysql') require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') +require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') +require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do @@ -8,4 +10,6 @@ task add_limits_mysql: :environment do LimitsToMysql.new.up MarkdownCacheLimitsToMysql.new.up MergeRequestDiffFileLimitsToMysql.new.up + LimitsCiBuildTraceChunksRawDataForMysql.new.up + IncreaseMysqlTextLimitForGpgKeys.new.up end diff --git a/lib/tasks/migrate/composite_primary_keys.rake b/lib/tasks/migrate/composite_primary_keys.rake new file mode 100644 index 00000000000..eb112434dd9 --- /dev/null +++ b/lib/tasks/migrate/composite_primary_keys.rake @@ -0,0 +1,15 @@ +namespace :gitlab do + namespace :db do + desc 'GitLab | Adds primary keys to tables that only have composite unique keys' + task composite_primary_keys_add: :environment do + require Rails.root.join('db/optional_migrations/composite_primary_keys') + CompositePrimaryKeysMigration.new.up + end + + desc 'GitLab | Removes previously added composite primary keys' + task composite_primary_keys_drop: :environment do + require Rails.root.join('db/optional_migrations/composite_primary_keys') + CompositePrimaryKeysMigration.new.down + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 1c7a8a90f5c..f69d204c579 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -7,8 +7,9 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') - require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') + require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') + require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,6 +18,22 @@ task setup_postgresql: :environment do AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up - ReworkRedirectRoutesIndexes.new.up UsersNameLowerIndex.new.up + ProjectNameLowerIndex.new.up + AddPathIndexToRedirectRoutes.new.up +end + +desc 'GitLab | Generate PostgreSQL Password Hash' +task :postgresql_md5_hash do + require 'digest' + username = ENV.fetch('USERNAME') do |missing| + puts "You must provide an username with '#{missing}' ENV variable" + exit(1) + end + password = ENV.fetch('PASSWORD') do |missing| + puts "You must provide a password with '#{missing}' ENV variable" + exit(1) + end + hash = Digest::MD5.hexdigest("#{password}#{username}") + puts "The MD5 hash of your database password for user: #{username} -> #{hash}" end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake deleted file mode 100644 index 19ff13f06c0..00000000000 --- a/lib/tasks/spinach.rake +++ /dev/null @@ -1,60 +0,0 @@ -Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach') - -namespace :spinach do - namespace :project do - desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" - task :half do - run_spinach_tests('@project_commits,@project_issues,@project_merge_requests') - end - - desc "GitLab | Spinach | Run remaining project spinach features" - task :rest do - run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests') - end - end - - desc "GitLab | Spinach | Run project spinach features" - task :project do - run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets') - end - - desc "GitLab | Spinach | Run other spinach features" - task :other do - run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets') - end - - desc "GitLab | Spinach | Run other spinach features" - task :builds do - run_spinach_tests('@builds') - end -end - -desc "GitLab | Run spinach" -task :spinach do - run_spinach_tests(nil) -end - -def run_system_command(cmd) - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) -end - -def run_spinach_command(args) - run_system_command(%w(spinach -r rerun) + args) -end - -def run_spinach_tests(tags) - success = run_spinach_command(%W(--tags #{tags})) - 3.times do |_| - break if success - break unless File.exist?('tmp/spinach-rerun.txt') - - tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) - puts '' - puts "Spinach tests for #{tags}: Retrying tests... #{tests}".color(:red) - puts '' - sleep(3) - success = run_spinach_command(tests) - end - - raise("spinach tests for #{tags} failed!") unless success -end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 3e01f91d32c..b52af81fc16 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -4,8 +4,3 @@ desc "GitLab | Run all tests" task :test do Rake::Task["gitlab:test"].invoke end - -unless Rails.env.production? - desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec] -end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 693597afdf8..81829668de8 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -6,9 +6,9 @@ namespace :tokens do reset_all_users_token(:reset_incoming_email_token!) end - desc "Reset all GitLab RSS tokens" - task reset_all_rss: :environment do - reset_all_users_token(:reset_rss_token!) + desc "Reset all GitLab feed tokens" + task reset_all_feed: :environment do + reset_all_users_token(:reset_feed_token!) end def reset_all_users_token(reset_token_method) @@ -31,8 +31,8 @@ class TmpUser < ActiveRecord::Base save!(validate: false) end - def reset_rss_token! - write_new_token(:rss_token) + def reset_feed_token! + write_new_token(:feed_token) save!(validate: false) end end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 4a3c40f88eb..5dc85b2baea 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -1,8 +1,10 @@ require "tempfile" +require "tmpdir" require "fileutils" -# Taken from: Rack::Test::UploadedFile class UploadedFile + InvalidPathError = Class.new(StandardError) + # The filename, *not* including the path, of the "uploaded" file attr_reader :original_filename @@ -12,14 +14,46 @@ class UploadedFile # The content type of the "uploaded" file attr_accessor :content_type - def initialize(path, filename, content_type = "text/plain") - raise "#{path} file does not exist" unless ::File.exist?(path) + attr_reader :remote_id + attr_reader :sha256 + + def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil) + raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path) @content_type = content_type @original_filename = filename || ::File.basename(path) + @content_type = content_type + @sha256 = sha256 + @remote_id = remote_id @tempfile = File.new(path, 'rb') end + def self.from_params(params, field, upload_path) + unless params["#{field}.path"] + raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"] + + return + end + + file_path = File.realpath(params["#{field}.path"]) + + unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact) + raise InvalidPathError, "insecure path used '#{file_path}'" + end + + UploadedFile.new(file_path, + filename: params["#{field}.name"], + content_type: params["#{field}.type"] || 'application/octet-stream', + sha256: params["#{field}.sha256"], + remote_id: params["#{field}.remote_id"]) + end + + def self.allowed_path?(file_path, paths) + paths.any? do |path| + File.exist?(path) && file_path.start_with?(File.realpath(path)) + end + end + def path @tempfile.path end |