diff options
Diffstat (limited to 'lib')
210 files changed, 5089 insertions, 2673 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb new file mode 100644 index 00000000000..d02b469dac8 --- /dev/null +++ b/lib/api/access_requests.rb @@ -0,0 +1,90 @@ +module API + class AccessRequests < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + resource source_type.pluralize do + # Get a list of group/project access requests viewable by the authenticated user. + # + # Parameters: + # id (required) - The group/project ID + # + # Example Request: + # GET /groups/:id/access_requests + # GET /projects/:id/access_requests + get ":id/access_requests" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + + access_requesters = paginate(source.requesters.includes(:user)) + + present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters + end + + # Request access to the group/project + # + # Parameters: + # id (required) - The group/project ID + # + # Example Request: + # POST /groups/:id/access_requests + # POST /projects/:id/access_requests + post ":id/access_requests" do + source = find_source(source_type, params[:id]) + access_requester = source.request_access(current_user) + + if access_requester.persisted? + present access_requester.user, with: Entities::AccessRequester, access_requester: access_requester + else + render_validation_error!(access_requester) + end + end + + # Approve a group/project access request + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the access requester + # access_level (optional) - Access level + # + # Example Request: + # PUT /groups/:id/access_requests/:user_id/approve + # PUT /projects/:id/access_requests/:user_id/approve + put ':id/access_requests/:user_id/approve' do + required_attributes! [:user_id] + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + + member = source.requesters.find_by!(user_id: params[:user_id]) + if params[:access_level] + member.update(access_level: params[:access_level]) + end + member.accept_request + + status :created + present member.user, with: Entities::Member, member: member + end + + # Deny a group/project access request + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the access requester + # + # Example Request: + # DELETE /groups/:id/access_requests/:user_id + # DELETE /projects/:id/access_requests/:user_id + delete ":id/access_requests/:user_id" do + required_attributes! [:user_id] + source = find_source(source_type, params[:id]) + + access_requester = source.requesters.find_by!(user_id: params[:user_id]) + + ::Members::DestroyService.new(access_requester, current_user).execute + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 3d7d67510a8..4602e627fdb 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,49 +3,54 @@ module API include APIGuard version 'v3', using: :path + rescue_from Gitlab::Access::AccessDeniedError do + rack_response({ 'message' => '403 Forbidden' }.to_json, 403) + end + rescue_from ActiveRecord::RecordNotFound do rack_response({ 'message' => '404 Not found' }.to_json, 404) end - rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") + # Retain 405 error rather than a 500 error for Grape 0.15.0+. + # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de + rescue_from Grape::Exceptions::Base do |e| + error! e.message, e.status, e.headers + end - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + rescue_from :all do |exception| + handle_api_exception(exception) end format :json content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::SentryHelper helpers ::API::Helpers + mount ::API::AccessRequests mount ::API::AwardEmoji mount ::API::Branches mount ::API::Builds mount ::API::CommitStatuses mount ::API::Commits mount ::API::DeployKeys + mount ::API::Deployments + mount ::API::Environments mount ::API::Files - mount ::API::GroupMembers mount ::API::Groups mount ::API::Internal mount ::API::Issues mount ::API::Keys mount ::API::Labels mount ::API::LicenseTemplates + mount ::API::Members mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes + mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectMembers mount ::API::ProjectSnippets mount ::API::Projects mount ::API::Repositories @@ -62,5 +67,6 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::MergeRequestDiffs end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index c4fa1838b5a..2efe7e3adf3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -56,9 +56,9 @@ module API not_found!('Award Emoji') unless can_read_awardable? - award = awardable.award_emoji.new(name: params[:name], user: current_user) + award = awardable.create_award_emoji(params[:name], current_user) - if award.save + if award.persisted? present award, with: Entities::AwardEmoji else not_found!("Award Emoji #{award.errors.messages}") diff --git a/lib/api/branches.rb b/lib/api/branches.rb index d467eb9d474..b615703df93 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -15,7 +15,8 @@ module API # GET /projects/:id/repository/branches get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) - present branches, with: Entities::RepoObject, project: user_project + + present branches, with: Entities::RepoBranch, project: user_project end # Get a single branch @@ -28,14 +29,21 @@ module API get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } not_found!("Branch") unless @branch - present @branch, with: Entities::RepoObject, project: user_project + + present @branch, with: Entities::RepoBranch, project: user_project end # Protect a single branch # + # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` + # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), + # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. + # # Parameters: # id (required) - The ID of a project # branch (required) - The name of the branch + # developers_can_push (optional) - Flag if developers can push to that branch + # developers_can_merge (optional) - Flag if developers can merge to that branch # Example Request: # PUT /projects/:id/repository/branches/:branch/protect put ':id/repository/branches/:branch/protect', @@ -43,11 +51,46 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless @branch + not_found!('Branch') unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) - user_project.protected_branches.create(name: @branch.name) unless protected_branch - present @branch, with: Entities::RepoObject, project: user_project + developers_can_merge = to_boolean(params[:developers_can_merge]) + developers_can_push = to_boolean(params[:developers_can_push]) + + protected_branch_params = { + name: @branch.name + } + + # If `developers_can_merge` is switched off, _all_ `DEVELOPER` + # merge_access_levels need to be deleted. + if developers_can_merge == false + protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all + end + + # If `developers_can_push` is switched off, _all_ `DEVELOPER` + # push_access_levels need to be deleted. + if developers_can_push == false + protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all + end + + protected_branch_params.merge!( + merge_access_levels_attributes: [{ + access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + }], + push_access_levels_attributes: [{ + access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + }] + ) + + if protected_branch + service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) + service.execute(protected_branch) + else + service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) + service.execute + end + + present @branch, with: Entities::RepoBranch, project: user_project end # Unprotect a single branch @@ -66,7 +109,7 @@ module API protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch - present @branch, with: Entities::RepoObject, project: user_project + present @branch, with: Entities::RepoBranch, project: user_project end # Create branch @@ -84,7 +127,7 @@ module API if result[:status] == :success present result[:branch], - with: Entities::RepoObject, + with: Entities::RepoBranch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/builds.rb b/lib/api/builds.rb index d36047acd1f..52bdbcae5a8 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -52,8 +52,7 @@ module API get ':id/builds/:build_id' do authorize_read_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) @@ -69,18 +68,27 @@ module API get ':id/builds/:build_id/artifacts' do authorize_read_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) - artifacts_file = build.artifacts_file + present_artifacts!(build.artifacts_file) + end - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end + # Download the artifacts file from ref_name and job + # + # Parameters: + # id (required) - The ID of a project + # ref_name (required) - The ref from repository + # job (required) - The name for the build + # Example Request: + # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + get ':id/builds/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! - return not_found! unless artifacts_file.exists? + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) - present_file!(artifacts_file.path, artifacts_file.filename) + present_artifacts!(latest_build.artifacts_file) end # Get a trace of a specific build of a project @@ -97,8 +105,7 @@ module API get ':id/builds/:build_id/trace' do authorize_read_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -118,8 +125,7 @@ module API post ':id/builds/:build_id/cancel' do authorize_update_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) build.cancel @@ -137,8 +143,7 @@ module API post ':id/builds/:build_id/retry' do authorize_update_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) return forbidden!('Build is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -157,8 +162,7 @@ module API post ':id/builds/:build_id/erase' do authorize_update_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build + build = get_build!(params[:build_id]) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -176,8 +180,8 @@ module API post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! - build = get_build(params[:build_id]) - return not_found!(build) unless build && build.artifacts? + build = get_build!(params[:build_id]) + return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -185,6 +189,27 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) end + + desc 'Trigger a manual build' do + success Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Build") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do @@ -192,6 +217,20 @@ module API user_project.builds.find_by(id: id.to_i) end + def get_build!(id) + get_build(id) || not_found! + end + + def present_artifacts!(artifacts_file) + if !artifacts_file.file_storage? + redirect_to(build.artifacts_file.url) + elsif artifacts_file.exists? + present_file!(artifacts_file.path, artifacts_file.filename) + else + not_found! + end + end + def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 323a7086890..4df6ca8333e 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -24,7 +24,7 @@ module API pipelines = user_project.pipelines.where(sha: params[:sha]) statuses = ::CommitStatus.where(pipeline: pipelines) - statuses = statuses.latest unless parse_boolean(params[:all]) + statuses = statuses.latest unless to_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? statuses = statuses.where(stage: params[:stage]) if params[:stage].present? statuses = statuses.where(name: params[:name]) if params[:name].present? @@ -64,7 +64,7 @@ module API ref = branches.first end - pipeline = @project.ensure_pipeline(commit.sha, ref) + pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) name = params[:name] || params[:context] status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4a11c8e3620..b4eaf1813d4 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -54,7 +54,7 @@ module API sha = params[:sha] commit = user_project.commit(sha) not_found! "Commit" unless commit - commit.diffs.to_a + commit.raw_diffs.to_a end # Get a commit's comments @@ -96,7 +96,7 @@ module API } if params[:path] && params[:line] && params[:line_type] - commit.diffs(all_diffs: true).each do |diff| + commit.raw_diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 06eb7756841..825e05fbae3 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -2,73 +2,114 @@ module API # Projects API class DeployKeys < Grape::API before { authenticate! } - before { authorize_admin_project } + get "deploy_keys" do + authenticated_as_admin! + + keys = DeployKey.all + present keys, with: Entities::SSHKey + end + + params do + requires :id, type: String, desc: 'The ID of the project' + end resource :projects do - # Get a specific project's keys - # - # Example Request: - # GET /projects/:id/keys - get ":id/keys" do - present user_project.deploy_keys, with: Entities::SSHKey - end + before { authorize_admin_project } - # Get single key owned by currently authenticated user + # Routing "projects/:id/keys/..." is DEPRECATED and WILL BE REMOVED in version 9.0 + # Use "projects/:id/deploy_keys/..." instead. # - # Example Request: - # GET /projects/:id/keys/:id - get ":id/keys/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - present key, with: Entities::SSHKey - end + %w(keys deploy_keys).each do |path| + desc "Get a specific project's deploy keys" do + success Entities::SSHKey + end + get ":id/#{path}" do + present user_project.deploy_keys, with: Entities::SSHKey + end - # Add new ssh key to currently authenticated user - # If deploy key already exists - it will be joined to project - # but only if original one was is accessible by same user - # - # Parameters: - # key (required) - New SSH Key - # title (required) - New SSH Key's title - # Example Request: - # POST /projects/:id/keys - post ":id/keys" do - attrs = attributes_for_keys [:title, :key] + desc 'Get single deploy key' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + get ":id/#{path}/:key_id" do + key = user_project.deploy_keys.find params[:key_id] + present key, with: Entities::SSHKey + end - if attrs[:key].present? - attrs[:key].strip! + # TODO: for 9.0 we should check if params are there with the params block + # grape provides, at this point we'd change behaviour so we can't + # Behaviour now if you don't provide all required params: it renders a + # validation error or two. + desc 'Add new deploy key to currently authenticated user' do + success Entities::SSHKey + end + post ":id/#{path}" do + attrs = attributes_for_keys [:title, :key] + attrs[:key].strip! if attrs[:key] - # check if key already exist in project key = user_project.deploy_keys.find_by(key: attrs[:key]) - if key - present key, with: Entities::SSHKey - return - end + present key, with: Entities::SSHKey if key # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey - return + end + + key = DeployKey.new attrs + + if key.valid? && user_project.deploy_keys << key + present key, with: Entities::SSHKey + else + render_validation_error!(key) end end - key = DeployKey.new attrs + desc 'Enable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + post ":id/#{path}/:key_id/enable" do + key = ::Projects::EnableDeployKeyService.new(user_project, + current_user, declared(params)).execute - if key.valid? && user_project.deploy_keys << key - present key, with: Entities::SSHKey - else - render_validation_error!(key) + if key + present key, with: Entities::SSHKey + else + not_found!('Deploy Key') + end end - end - # Delete existed ssh key of currently authenticated user - # - # Example Request: - # DELETE /projects/:id/keys/:id - delete ":id/keys/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - key.destroy + desc 'Disable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id/disable" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + key.destroy + + present key.deploy_key, with: Entities::SSHKey + end + + desc 'Delete existing deploy key of currently authenticated user' do + success Key + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id" do + key = user_project.deploy_keys.find(params[:key_id]) + key.destroy + end end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb new file mode 100644 index 00000000000..f782bcaf7e9 --- /dev/null +++ b/lib/api/deployments.rb @@ -0,0 +1,40 @@ +module API + # Deployments RESTfull API endpoints + class Deployments < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: Entities::Deployment + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: Entities::Deployment + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9076a0c3831..cbb324dd06d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -48,12 +48,14 @@ module API class ProjectHook < Hook expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :issues_events, :merge_requests_events, :tag_push_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events expose :enable_ssl_verification end class BasicProjectDetails < Grape::Entity expose :id + expose :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace end @@ -88,11 +90,24 @@ module API expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end + expose :only_allow_merge_if_build_succeeds end - class ProjectMember < UserBasic + class Member < UserBasic expose :access_level do |user, options| - options[:project].project_members.find_by(user_id: user.id).access_level + member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member.access_level + end + expose :expires_at do |user, options| + member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member.expires_at + end + end + + class AccessRequester < UserBasic + expose :requested_at do |user, options| + access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id } + access_requester.requested_at end end @@ -107,27 +122,27 @@ module API expose :shared_projects, using: Entities::Project end - class GroupMember < UserBasic - expose :access_level do |user, options| - options[:group].group_members.find_by(user_id: user.id).access_level + class RepoBranch < Grape::Entity + expose :name + + expose :commit do |repo_branch, options| + options[:project].repository.commit(repo_branch.target) end - end - class RepoObject < Grape::Entity - expose :name + expose :protected do |repo_branch, options| + options[:project].protected_branch? repo_branch.name + end - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end + expose :developers_can_push do |repo_branch, options| + project = options[:project] + access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end - expose :protected do |repo, options| - if options[:project] - options[:project].protected_branch? repo.name - end + expose :developers_can_merge do |repo_branch, options| + project = options[:project] + access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end end @@ -146,8 +161,13 @@ module API expose :safe_message, as: :message end + class RepoCommitStats < Grape::Entity + expose :additions, :deletions, :total + end + class RepoCommitDetail < RepoCommit expose :parent_ids, :committed_date, :authored_date + expose :stats, using: Entities::RepoCommitStats expose :status end @@ -158,6 +178,10 @@ module API # TODO (rspeicher): Deprecated; remove in 9.0 expose(:expires_at) { |snippet| nil } + + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end end class ProjectEntity < Grape::Entity @@ -186,6 +210,11 @@ module API end expose :user_notes_count expose :upvotes, :downvotes + expose :due_date + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end end class ExternalIssue < Grape::Entity @@ -199,7 +228,6 @@ module API expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels - expose :description expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds @@ -208,11 +236,30 @@ module API merge_request.subscribed?(options[:current_user]) end expose :user_notes_count + expose :should_remove_source_branch?, as: :should_remove_source_branch + expose :force_remove_source_branch?, as: :force_remove_source_branch + + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end end class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.diffs(all_diffs: true).to_a + compare.raw_diffs(all_diffs: true).to_a + end + end + + class MergeRequestDiff < Grape::Entity + expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha, + :created_at, :merge_request_id, :state, :real_size + end + + class MergeRequestDiffFull < MergeRequestDiff + expose :commits, using: Entities::RepoCommit + + expose :diffs, using: Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a end end @@ -313,7 +360,7 @@ module API expose :id, :path, :kind end - class Member < Grape::Entity + class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| if member.notification_setting @@ -322,15 +369,16 @@ module API end end - class ProjectAccess < Member + class ProjectAccess < MemberAccess end - class GroupAccess < Member + class GroupAccess < MemberAccess end class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :push_events, :issues_events, :merge_requests_events + expose :tag_push_events, :note_events, :build_events, :pipeline_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -409,7 +457,9 @@ module API expose :default_project_visibility expose :default_snippet_visibility expose :default_group_visibility - expose :restricted_signup_domains + expose :domain_whitelist + expose :domain_blacklist_enabled + expose :domain_blacklist expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay @@ -422,27 +472,14 @@ module API end class RepoTag < Grape::Entity - expose :name - expose :message do |repo_obj, _options| - if repo_obj.respond_to?(:message) - repo_obj.message - else - nil - end - end + expose :name, :message - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end + expose :commit do |repo_tag, options| + options[:project].repository.commit(repo_tag.target) end - expose :release, using: Entities::Release do |repo_obj, options| - if options[:project] - options[:project].releases.find_by(tag: repo_obj.name) - end + expose :release, using: Entities::Release do |repo_tag, options| + options[:project].releases.find_by(tag: repo_tag.name) end end @@ -495,6 +532,29 @@ module API expose :key, :value end + class Pipeline < Grape::Entity + expose :id, :status, :ref, :sha, :before_sha, :tag, :yaml_errors + + expose :user, with: Entities::UserBasic + expose :created_at, :updated_at, :started_at, :finished_at, :committed_at + expose :duration + end + + class EnvironmentBasic < Grape::Entity + expose :id, :name, :external_url + end + + class Environment < EnvironmentBasic + expose :project, using: Entities::Project + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: Entities::UserBasic + expose :environment, using: Entities::EnvironmentBasic + expose :deployable, using: Entities::Build + end + class RepoLicense < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular diff --git a/lib/api/environments.rb b/lib/api/environments.rb new file mode 100644 index 00000000000..819f80d8365 --- /dev/null +++ b/lib/api/environments.rb @@ -0,0 +1,83 @@ +module API + # Environments RESTfull API endpoints + class Environments < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all environments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + get ':id/environments' do + authorize! :read_environment, user_project + + present paginate(user_project.environments), with: Entities::Environment + end + + desc 'Creates a new environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :name, type: String, desc: 'The name of the environment to be created' + optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + end + post ':id/environments' do + authorize! :create_environment, user_project + + create_params = declared(params, include_parent_namespaces: false).to_h + environment = user_project.environments.create(create_params) + + if environment.persisted? + present environment, with: Entities::Environment + else + render_validation_error!(environment) + end + end + + desc 'Updates an existing environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :environment_id, type: Integer, desc: 'The environment ID' + optional :name, type: String, desc: 'The new environment name' + optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + end + put ':id/environments/:environment_id' do + authorize! :update_environment, user_project + + environment = user_project.environments.find(params[:environment_id]) + + update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h + if environment.update(update_params) + present environment, with: Entities::Environment + else + render_validation_error!(environment) + end + end + + desc 'Deletes an existing environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :environment_id, type: Integer, desc: 'The environment ID' + end + delete ':id/environments/:environment_id' do + authorize! :update_environment, user_project + + environment = user_project.environments.find(params[:environment_id]) + + present environment.destroy, with: Entities::Environment + end + end + end +end diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb deleted file mode 100644 index dbe5bb08d3f..00000000000 --- a/lib/api/group_members.rb +++ /dev/null @@ -1,87 +0,0 @@ -module API - class GroupMembers < Grape::API - before { authenticate! } - - resource :groups do - # Get a list of group members viewable by the authenticated user. - # - # Example Request: - # GET /groups/:id/members - get ":id/members" do - group = find_group(params[:id]) - users = group.users - present users, with: Entities::GroupMember, group: group - end - - # Add a user to the list of group members - # - # Parameters: - # id (required) - group id - # user_id (required) - the users id - # access_level (required) - Project access level - # Example Request: - # POST /groups/:id/members - post ":id/members" do - group = find_group(params[:id]) - authorize! :admin_group, group - required_attributes! [:user_id, :access_level] - - unless validate_access_level?(params[:access_level]) - render_api_error!("Wrong access level", 422) - end - - if group.group_members.find_by(user_id: params[:user_id]) - render_api_error!("Already exists", 409) - end - - group.add_users([params[:user_id]], params[:access_level], current_user) - member = group.group_members.find_by(user_id: params[:user_id]) - present member.user, with: Entities::GroupMember, group: group - end - - # Update group member - # - # Parameters: - # id (required) - The ID of a group - # user_id (required) - The ID of a group member - # access_level (required) - Project access level - # Example Request: - # PUT /groups/:id/members/:user_id - put ':id/members/:user_id' do - group = find_group(params[:id]) - authorize! :admin_group, group - required_attributes! [:access_level] - - group_member = group.group_members.find_by(user_id: params[:user_id]) - not_found!('User can not be found') if group_member.nil? - - if group_member.update_attributes(access_level: params[:access_level]) - @member = group_member.user - present @member, with: Entities::GroupMember, group: group - else - handle_member_errors group_member.errors - end - end - - # Remove member. - # - # Parameters: - # id (required) - group id - # user_id (required) - the users id - # - # Example Request: - # DELETE /groups/:id/members/:user_id - delete ":id/members/:user_id" do - group = find_group(params[:id]) - authorize! :admin_group, group - member = group.group_members.find_by(user_id: params[:user_id]) - - if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) - else - member.destroy - end - end - end - end -end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 77e407b54c5..da4b1bf9902 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -5,8 +5,11 @@ module API SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def parse_boolean(value) - [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) + def to_boolean(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + nil end def find_user_by_private_token @@ -17,7 +20,7 @@ module API def current_user @current_user ||= (find_user_by_private_token || doorkeeper_guard) - unless @current_user && Gitlab::UserAccess.allowed?(@current_user) + unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil end @@ -25,7 +28,7 @@ module API # If the sudo is the current user do nothing if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? + forbidden!('Must be admin to use sudo') unless @current_user.is_admin? @current_user = User.by_username_or_id(identifier) not_found!("No user id or username for: #{identifier}") if @current_user.nil? end @@ -46,16 +49,15 @@ module API def user_project @project ||= find_project(params[:id]) - @project || not_found!("Project") end def find_project(id) project = Project.find_with_namespace(id) || Project.find_by(id: id) - if project && can?(current_user, :read_project, project) + if can?(current_user, :read_project, project) project else - nil + not_found!('Project') end end @@ -86,11 +88,7 @@ module API end def find_group(id) - begin - group = Group.find(id) - rescue ActiveRecord::RecordNotFound - group = Group.find_by!(path: id) - end + group = Group.find_by(path: id) || Group.find_by(id: id) if can?(current_user, :read_group, group) group @@ -132,7 +130,7 @@ module API end def authorize!(action, subject) - forbidden! unless abilities.allowed?(current_user, action, subject) + forbidden! unless can?(current_user, action, subject) end def authorize_push_project @@ -194,10 +192,6 @@ module API errors end - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. # @@ -285,12 +279,30 @@ module API error!({ 'message' => message }, status) end + def handle_api_exception(exception) + if sentry_enabled? && report_exception?(exception) + define_params_for_grape_middleware + sentry_context + Raven.capture_exception(exception) + end + + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + end + # Projects helpers def filter_projects(projects) # If the archived parameter is passed, limit results accordingly if params[:archived].present? - projects = projects.where(archived: parse_boolean(params[:archived])) + projects = projects.where(archived: to_boolean(params[:archived])) end if params[:search].present? @@ -408,11 +420,6 @@ module API File.read(Gitlab.config.gitlab_shell.secret_file).chomp end - def handle_member_errors(errors) - error!(errors[:access_level], 422) if errors[:access_level].any? - not_found!(errors) - end - def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' @@ -430,5 +437,19 @@ module API Entities::Issue end end + + # The Grape Error Middleware only has access to env but no params. We workaround this by + # defining a method that returns the right value. + def define_params_for_grape_middleware + self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + end + + # We could get a Grape or a standard Ruby exception. We should only report anything that + # is clearly an error. + def report_exception?(exception) + return true unless exception.respond_to?(:status) + + exception.status == 500 + end end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb new file mode 100644 index 00000000000..90114f6f667 --- /dev/null +++ b/lib/api/helpers/members_helpers.rb @@ -0,0 +1,13 @@ +module API + module Helpers + module MembersHelpers + def find_source(source_type, id) + public_send("find_#{source_type}", id) + end + + def authorize_admin_source!(source_type, source) + authorize! :"admin_#{source_type}", source + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index d5dfba5e0cc..5b54c11ef62 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -63,12 +63,21 @@ module API if access_status.status # Return the repository full path so that gitlab-shell has it when # handling ssh commands - response[:repository_path] = project.repository.path_to_repo + response[:repository_path] = + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end end response end + get "/merge_request_urls" do + ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + end + # # Discover user by ssh key # @@ -92,6 +101,31 @@ module API {} end end + + post '/two_factor_recovery_codes' do + status 200 + + key = Key.find(params[:key_id]) + user = key.user + + # Make sure this isn't a deploy key + unless key.type.nil? + return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + unless user.present? + return { success: false, message: 'Could not find a user for the given key' } + end + + unless user.two_factor_enabled? + return { success: false, message: 'Two-factor authentication is not enabled for this user' } + end + + codes = user.generate_otp_backup_codes! + user.save! + + { success: true, recovery_codes: codes } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 8a03a41e9c5..077258faee1 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,8 +3,6 @@ module API class Issues < Grape::API before { authenticate! } - helpers ::Gitlab::AkismetHelper - helpers do def filter_issues_state(issues, state) case state @@ -21,17 +19,6 @@ module API def filter_issues_milestone(issues, milestone) issues.includes(:milestone).where('milestones.title' => milestone) end - - def create_spam_log(project, current_user, attrs) - params = attrs.merge({ - source_ip: client_ip(env), - user_agent: user_agent(env), - noteable_type: 'Issue', - via_api: true - }) - - ::CreateSpamLogService.new(project, current_user, params).execute - end end resource :issues do @@ -152,12 +139,13 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # POST /projects/:id/issues - post ":id/issues" do + post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -167,15 +155,13 @@ module API end project = user_project - text = [attrs[:title], attrs[:description]].reject(&:blank?).join("\n") - if check_for_spam?(project, current_user) && is_spam?(env, current_user, text) - create_spam_log(project, current_user, attrs) + issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute + + if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end - issue = ::Issues::CreateService.new(project, current_user, attrs).execute - if issue.valid? # Find or create labels and attach to issue. Labels are valid because # we already checked its name, so there can't be an error here @@ -201,12 +187,13 @@ module API # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # PUT /projects/:id/issues/:issue_id - put ":id/issues/:issue_id" do + put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) diff --git a/lib/api/members.rb b/lib/api/members.rb new file mode 100644 index 00000000000..94c16710d9a --- /dev/null +++ b/lib/api/members.rb @@ -0,0 +1,158 @@ +module API + class Members < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + resource source_type.pluralize do + # Get a list of group/project members viewable by the authenticated user. + # + # Parameters: + # id (required) - The group/project ID + # query - Query string + # + # Example Request: + # GET /groups/:id/members + # GET /projects/:id/members + get ":id/members" do + source = find_source(source_type, params[:id]) + + members = source.members.includes(:user) + members = members.joins(:user).merge(User.search(params[:query])) if params[:query] + members = paginate(members) + + present members.map(&:user), with: Entities::Member, members: members + end + + # Get a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # + # Example Request: + # GET /groups/:id/members/:user_id + # GET /projects/:id/members/:user_id + get ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + members = source.members + member = members.find_by!(user_id: params[:user_id]) + + present member.user, with: Entities::Member, member: member + end + + # Add a new group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the new member + # access_level (required) - A valid access level + # expires_at (optional) - Date string in the format YEAR-MONTH-DAY + # + # Example Request: + # POST /groups/:id/members + # POST /projects/:id/members + post ":id/members" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + required_attributes! [:user_id, :access_level] + + access_requester = source.requesters.find_by(user_id: params[:user_id]) + if access_requester + # We pass current_user = access_requester so that the requester doesn't + # receive a "access denied" email + ::Members::DestroyService.new(access_requester, access_requester.user).execute + end + + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but 409 behavior should be used + # for both project and group members in 9.0! + conflict!('Member already exists') if source_type == 'group' && member + + unless member + source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + member = source.members.find_by(user_id: params[:user_id]) + end + + if member + present member.user, with: Entities::Member, member: member + else + # Since `source.add_user` doesn't return a member object, we have to + # build a new one and populate its errors in order to render them. + member = source.members.build(attributes_for_keys([:user_id, :access_level, :expires_at])) + member.valid? # populate the errors + + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + # Update a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # access_level (required) - A valid access level + # expires_at (optional) - Date string in the format YEAR-MONTH-DAY + # + # Example Request: + # PUT /groups/:id/members/:user_id + # PUT /projects/:id/members/:user_id + put ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + required_attributes! [:user_id, :access_level] + + member = source.members.find_by!(user_id: params[:user_id]) + attrs = attributes_for_keys [:access_level, :expires_at] + + if member.update_attributes(attrs) + present member.user, with: Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + # Remove a group/project member + # + # Parameters: + # id (required) - The group/project ID + # user_id (required) - The user ID of the member + # + # Example Request: + # DELETE /groups/:id/members/:user_id + # DELETE /projects/:id/members/:user_id + delete ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + required_attributes! [:user_id] + + # This is to ensure back-compatibility but find_by! should be used + # in that casse in 9.0! + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but this should be removed in + # favor of find_by! in 9.0! + not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? + + # This is to ensure back-compatibility but 204 behavior should be used + # for all DELETE endpoints in 9.0! + if member.nil? + { message: "Access revoked", id: params[:user_id].to_i } + else + ::Members::DestroyService.new(member, current_user).execute + + present member.user, with: Entities::Member, member: member + end + end + end + end + end +end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb new file mode 100644 index 00000000000..07435d78468 --- /dev/null +++ b/lib/api/merge_request_diffs.rb @@ -0,0 +1,45 @@ +module API + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + before { authenticate! } + + resource :projects do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiff + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiffFull + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 4fcdf8968c9..2b685621da9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -242,7 +242,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? + if to_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). execute(merge_request) else diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb new file mode 100644 index 00000000000..2aae75c471d --- /dev/null +++ b/lib/api/pipelines.rb @@ -0,0 +1,74 @@ +module API + class Pipelines < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + present paginate(user_project.pipelines), with: Entities::Pipeline + end + + desc 'Gets a specific pipeline for the project' do + detail 'This feature was introduced in GitLab 8.11' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + get ':id/pipelines/:pipeline_id' do + authorize! :read_pipeline, user_project + + present pipeline, with: Entities::Pipeline + end + + desc 'Retry failed builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/retry' do + authorize! :update_pipeline, user_project + + pipeline.retry_failed(current_user) + + present pipeline, with: Entities::Pipeline + end + + desc 'Cancel all builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/cancel' do + authorize! :update_pipeline, user_project + + pipeline.cancel_running + + status 200 + present pipeline.reload, with: Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end +end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 6bb70bc8bc3..14f5be3b5f6 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -45,6 +45,8 @@ module API :tag_push_events, :note_events, :build_events, + :pipeline_events, + :wiki_page_events, :enable_ssl_verification ] @hook = user_project.hooks.new(attrs) @@ -78,6 +80,8 @@ module API :tag_push_events, :note_events, :build_events, + :pipeline_events, + :wiki_page_events, :enable_ssl_verification ] diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb deleted file mode 100644 index 6a0b3e7d134..00000000000 --- a/lib/api/project_members.rb +++ /dev/null @@ -1,110 +0,0 @@ -module API - # Projects members API - class ProjectMembers < Grape::API - before { authenticate! } - - resource :projects do - # Get a project team members - # - # Parameters: - # id (required) - The ID of a project - # query - Query string - # Example Request: - # GET /projects/:id/members - get ":id/members" do - if params[:query].present? - @members = paginate user_project.users.where("username LIKE ?", "%#{params[:query]}%") - else - @members = paginate user_project.users - end - present @members, with: Entities::ProjectMember, project: user_project - end - - # Get a project team members - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a user - # Example Request: - # GET /projects/:id/members/:user_id - get ":id/members/:user_id" do - @member = user_project.users.find params[:user_id] - present @member, with: Entities::ProjectMember, project: user_project - end - - # Add a new project team member - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a user - # access_level (required) - Project access level - # Example Request: - # POST /projects/:id/members - post ":id/members" do - authorize! :admin_project, user_project - required_attributes! [:user_id, :access_level] - - # either the user is already a team member or a new one - project_member = user_project.project_member(params[:user_id]) - if project_member.nil? - project_member = user_project.project_members.new( - user_id: params[:user_id], - access_level: params[:access_level] - ) - end - - if project_member.save - @member = project_member.user - present @member, with: Entities::ProjectMember, project: user_project - else - handle_member_errors project_member.errors - end - end - - # Update project team member - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a team member - # access_level (required) - Project access level - # Example Request: - # PUT /projects/:id/members/:user_id - put ":id/members/:user_id" do - authorize! :admin_project, user_project - required_attributes! [:access_level] - - project_member = user_project.project_members.find_by(user_id: params[:user_id]) - not_found!("User can not be found") if project_member.nil? - - if project_member.update_attributes(access_level: params[:access_level]) - @member = project_member.user - present @member, with: Entities::ProjectMember, project: user_project - else - handle_member_errors project_member.errors - end - end - - # Remove a team member from project - # - # Parameters: - # id (required) - The ID of a project - # user_id (required) - The ID of a team member - # Example Request: - # DELETE /projects/:id/members/:user_id - delete ":id/members/:user_id" do - project_member = user_project.project_members.find_by(user_id: params[:user_id]) - - unless current_user.can?(:admin_project, user_project) || - current_user.can?(:destroy_project_member, project_member) - forbidden! - end - - if project_member.nil? - { message: "Access revoked", id: params[:user_id].to_i } - else - project_member.destroy - end - end - end - end -end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0cc1edd65c8..71efd4f33ca 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -8,7 +8,7 @@ module API def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) if publik.present? && !attrs[:visibility_level].present? - publik = parse_boolean(publik) + publik = to_boolean(publik) # Since setting the public attribute to private could mean either # private or internal, use the more conservative option, private. attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE @@ -25,7 +25,11 @@ module API @projects = current_user.authorized_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + if params[:simple] + present @projects, with: Entities::BasicProjectDetails, user: current_user + else + present @projects, with: Entities::ProjectWithAccess, user: current_user + end end # Get an owned projects list for authenticated user @@ -119,7 +123,8 @@ module API :public, :visibility_level, :import_url, - :public_builds] + :public_builds, + :only_allow_merge_if_build_succeeds] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -168,7 +173,8 @@ module API :public, :visibility_level, :import_url, - :public_builds] + :public_builds, + :only_allow_merge_if_build_succeeds] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -230,7 +236,8 @@ module API :shared_runners_enabled, :public, :visibility_level, - :public_builds] + :public_builds, + :only_allow_merge_if_build_succeeds] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -319,7 +326,7 @@ module API # DELETE /projects/:id delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).pending_delete! + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute end # Mark this project as forked from another diff --git a/lib/api/session.rb b/lib/api/session.rb index 56c202f1294..55ec66a6d67 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -14,6 +14,7 @@ module API user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user + return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? present user, with: Entities::UserLogin end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 18408797756..b9e718147e1 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,21 +1,28 @@ module API class Templates < Grape::API - TEMPLATE_TYPES = { - gitignores: Gitlab::Template::Gitignore, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYml + GLOBAL_TEMPLATE_TYPES = { + gitignores: Gitlab::Template::GitignoreTemplate, + gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate }.freeze - TEMPLATE_TYPES.each do |template, klass| + helpers do + def render_response(template_type, template) + not_found!(template_type.to_s.singularize) unless template + present template, with: Entities::Template + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, klass| # Get the list of the available template # # Example Request: # GET /gitignores # GET /gitlab_ci_ymls - get template.to_s do + get template_type.to_s do present klass.all, with: Entities::TemplatesList end - # Get the text for a specific template + # Get the text for a specific template present in local filesystem # # Parameters: # name (required) - The name of a template @@ -23,13 +30,10 @@ module API # Example Request: # GET /gitignores/Elixir # GET /gitlab_ci_ymls/Ruby - get "#{template}/:name" do + get "#{template_type}/:name" do required_attributes! [:name] - new_template = klass.find(params[:name]) - not_found!(template.to_s.singularize) unless new_template - - present new_template, with: Entities::Template + render_response(template_type, new_template) end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 2a6bfa98ca4..19df13d8aac 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -61,9 +61,9 @@ module API # delete ':id' do todo = current_user.todos.find(params[:id]) - todo.done + TodoService.new.mark_todos_as_done([todo], current_user) - present todo, with: Entities::Todo, current_user: current_user + present todo.reload, with: Entities::Todo, current_user: current_user end # Mark all todos as done @@ -73,9 +73,7 @@ module API # delete do todos = find_todos - todos.each(&:done) - - present paginate(Kaminari.paginate_array(todos)), with: Entities::Todo, current_user: current_user + TodoService.new.mark_todos_as_done(todos, current_user) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 654b4d1c896..cedbb289f6a 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -27,7 +27,7 @@ module Backup def backup_existing_files_dir timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") - if File.exists?(app_files_dir) + if File.exist?(app_files_dir) FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 2ff3e3bdfb0..0dfffaf0bc6 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -114,7 +114,7 @@ module Backup tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") - unless File.exists?(tar_file) + unless File.exist?(tar_file) puts "The specified backup doesn't exist!" exit 1 end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index b9773f98d75..f117fc3d37d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -28,7 +28,7 @@ module Backup wiki = ProjectWiki.new(project) - if File.exists?(path_to_repo(wiki)) + if File.exist?(path_to_repo(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) @@ -49,21 +49,21 @@ module Backup def restore Gitlab.config.repositories.storages.each do |name, path| - next unless File.exists?(path) + next unless File.exist?(path) # Move repos dir to 'repositories.old' dir bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) FileUtils.mv(path, bk_repos_path) + # This is expected from gitlab:check + FileUtils.mkdir_p(path, mode: 2770) end - FileUtils.mkdir_p(repos_path) - Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " project.ensure_dir_exist - if File.exists?(path_to_bundle(project)) + if File.exist?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) else @@ -80,7 +80,7 @@ module Backup wiki = ProjectWiki.new(project) - if File.exists?(path_to_bundle(wiki)) + if File.exist?(path_to_bundle(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo diff --git a/lib/banzai.rb b/lib/banzai.rb index 093382261ae..9ebe379f454 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.cache_collection_render(texts_and_contexts) + Renderer.cache_collection_render(texts_and_contexts) + end + def self.render_result(text, context = {}) Renderer.render_result(text, context) end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index fac7dad3243..799b83b1069 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -31,6 +31,14 @@ module Banzai # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set + # The XPath query to use for finding text nodes to parse. + TEXT_QUERY = %Q(descendant-or-self::text()[ + not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) + and contains(., '://') + and not(starts-with(., 'http')) + and not(starts-with(., 'ftp')) + ]) + def call return doc if context[:autolink] == false @@ -56,6 +64,8 @@ module Banzai # period (e.g., http://localhost:3000/) rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) + return if rinku == html + # Rinku returns a String, so parse it back to a Nokogiri::XML::Document # for further processing. @doc = parse_html(rinku) @@ -64,16 +74,11 @@ module Banzai # Autolinks any text matching LINK_PATTERN that Rinku didn't already # replace def text_parse - search_text_nodes(doc).each do |node| + doc.xpath(TEXT_QUERY).each do |node| content = node.to_html - next if has_ancestor?(node, IGNORE_PARENTS) next unless content.match(LINK_PATTERN) - # If Rinku didn't link this, there's probably a good reason, so we'll - # skip it too - next if content.start_with?(*%w(http https ftp)) - html = autolink_filter(content) next if html == content diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index ae7d31cf191..2492b5213ac 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -38,6 +38,11 @@ module Banzai end end + # Build a regexp that matches all valid :emoji: names. + def self.emoji_pattern + @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + end + private def emoji_url(name) @@ -59,11 +64,6 @@ module Banzai ActionController::Base.helpers.url_to_image(image) end - # Build a regexp that matches all valid :emoji: names. - def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ - end - def emoji_pattern self.class.emoji_pattern end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 9b209533a89..ff580ec68f8 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -12,7 +12,12 @@ module Banzai html end - private + def self.renderer + @renderer ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end def self.redcarpet_options # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use @@ -28,12 +33,7 @@ module Banzai }.freeze end - def self.renderer - @renderer ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) - end - end + private_class_method :redcarpet_options end end end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index c78da404607..4fa8d05481f 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -20,7 +20,7 @@ module Banzai process_link_attr el.attribute('href') end - doc.search('img').each do |el| + doc.css('img, video').each do |el| process_link_attr el.attribute('src') end @@ -35,6 +35,7 @@ module Banzai def process_link_attr(html_attr) return if html_attr.blank? + return if html_attr.value.start_with?('//') uri = URI(html_attr.value) if uri.relative? && uri.path.present? @@ -51,7 +52,7 @@ module Banzai relative_url_root, context[:project].path_with_namespace, uri_type(file_path), - ref || context[:project].default_branch, # if no ref exists, point to the default branch + ref, file_path ].compact.join('/').squeeze('/').chomp('/') @@ -87,10 +88,13 @@ module Banzai def build_relative_path(path, request_path) return request_path if path.empty? return path unless request_path + return path[1..-1] if path.start_with?('/') parts = request_path.split('/') parts.pop if uri_type(request_path) != :tree + path.sub!(%r{\A\./}, '') + while path.start_with?('../') parts.pop path.sub!('../', '') @@ -112,8 +116,7 @@ module Banzai end def current_commit - @current_commit ||= context[:commit] || - ref ? repository.commit(ref) : repository.head_commit + @current_commit ||= context[:commit] || repository.commit(ref) end def relative_url_root @@ -121,7 +124,7 @@ module Banzai end def ref - context[:ref] + context[:ref] || context[:project].default_branch end def repository diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index ca80aac5a08..6e13282d5f4 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -7,7 +7,7 @@ module Banzai UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze def whitelist - whitelist = super + whitelist = super.dup customize_whitelist(whitelist) @@ -42,6 +42,8 @@ module Banzai # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') + whitelist[:transformers] = whitelist[:transformers].dup + # ...but then remove links with unsafe protocols whitelist[:transformers].push(remove_unsafe_links) diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 536b478979f..fcdb496aed2 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -17,31 +17,48 @@ module Banzai def highlight_node(node) language = node.attr('class') - code = node.text + code = node.text + css_classes = "code highlight" + lexer = lexer_for(language) begin - highlighted = block_code(code, language) + code = format(lex(lexer, code)) + + css_classes << " js-syntax-highlight #{lexer.tag}" rescue # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. - highlighted = "<pre>#{code}</pre>" end + highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) end private + # Separate method so it can be instrumented. + def lex(lexer, code) + lexer.lex(code) + end + + def format(tokens) + rouge_formatter.format(tokens) + end + + def lexer_for(language) + (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new + end + def replace_parent_pre_element(node, highlighted) # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end # Override Rouge::Plugins::Redcarpet#rouge_formatter - def rouge_formatter(lexer) - Rouge::Formatters::HTMLGitlab.new( - cssclass: "code highlight js-syntax-highlight #{lexer.tag}") + def rouge_formatter(lexer = nil) + @rouge_formatter ||= Rouge::Formatters::HTML.new end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 5b0a6d8541b..e1ca7f4d24b 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -112,7 +112,7 @@ module Banzai data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text) + link_tag(url, data, text, 'All Project and Group Members') end def link_to_namespace(namespace, link_text: nil) @@ -128,7 +128,7 @@ module Banzai data = data_attribute(group: namespace.id) text = link_text || Group.reference_prefix + group - link_tag(url, data, text) + link_tag(url, data, text, namespace.name) end def link_to_user(user, namespace, link_text: nil) @@ -136,11 +136,11 @@ module Banzai data = data_attribute(user: namespace.owner_id) text = link_text || User.reference_prefix + user - link_tag(url, data, text) + link_tag(url, data, text, namespace.owner_name) end - def link_tag(url, data, text) - %(<a href="#{url}" #{data} class="#{link_class}">#{escape_once(text)}</a>) + def link_tag(url, data, text, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) end end end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb new file mode 100644 index 00000000000..ac7bbcb0d10 --- /dev/null +++ b/lib/banzai/filter/video_link_filter.rb @@ -0,0 +1,56 @@ +module Banzai + module Filter + # Find every image that isn't already wrapped in an `a` tag, and that has + # a `src` attribute ending with a video extension, add a new video node and + # a "Download" link in the case the video cannot be played. + class VideoLinkFilter < HTML::Pipeline::Filter + def call + doc.xpath(query).each do |el| + el.replace(video_node(doc, el)) + end + + doc + end + + private + + def query + @query ||= begin + src_query = UploaderHelper::VIDEO_EXT.map do |ext| + "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})" + end + + "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]" + end + end + + def video_node(doc, element) + container = doc.document.create_element( + 'div', + class: 'video-container' + ) + + video = doc.document.create_element( + 'video', + src: element['src'], + width: '400', + controls: true, + 'data-setup' => '{}') + + link = doc.document.create_element( + 'a', + element['title'] || element['alt'], + href: element['src'], + target: '_blank', + title: "Download '#{element['title'] || element['alt']}'") + download_paragraph = doc.document.create_element('p') + download_paragraph.children = link + + container.add_child(video) + container.add_child(download_paragraph) + + container + end + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index f0e4f28bf12..9aef807c152 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -31,17 +31,15 @@ module Banzai redacted = redact_documents(documents) objects.each_with_index do |object, index| - object.__send__("#{attribute}_html=", redacted.fetch(index)) + redacted_data = redacted[index] + object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.user_visible_reference_count = redacted_data[:visible_reference_count] end - - objects end # Renders the attribute of every given object. def render_objects(objects, attribute) - objects.map do |object| - render_attribute(object, attribute) - end + render_attributes(objects, attribute) end # Redacts the list of documents. @@ -50,9 +48,7 @@ module Banzai def redact_documents(documents) redactor = Redactor.new(project, user) - redactor.redact(documents).map do |document| - document.to_html.html_safe - end + redactor.redact(documents) end # Returns a Banzai context for the given object and attribute. @@ -66,16 +62,21 @@ module Banzai context end - # Renders the attribute of an object. + # Renders the attributes of a set of objects. # - # Returns a `Nokogiri::HTML::Document`. - def render_attribute(object, attribute) - context = context_for(object, attribute) + # Returns an Array of `Nokogiri::HTML::Document`. + def render_attributes(objects, attribute) + strings_and_contexts = objects.map do |object| + context = context_for(object, attribute) - string = object.__send__(attribute) - html = Banzai.render(string, context) + string = object.__send__(attribute) - Banzai::Pipeline[:relative_link].to_document(html, context) + { text: string, context: context } + end + + Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| + Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + end end def base_context diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index b27ecf3c923..8d94b199c66 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::UploadLinkFilter, + Filter::VideoLinkFilter, Filter::ImageLinkFilter, Filter::EmojiFilter, Filter::TableOfContentsFilter, diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index ffd267d5e9a..0df3a72d1c4 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -19,29 +19,36 @@ module Banzai # # Returns the documents passed as the first argument. def redact(documents) - nodes = documents.flat_map do |document| - Querying.css(document, 'a.gfm[data-reference-type]') - end - - redact_nodes(nodes) + all_document_nodes = document_nodes(documents) - documents + redact_document_nodes(all_document_nodes) end - # Redacts the given nodes + # Redacts the given node documents # - # nodes - An Array of HTML nodes to redact. - def redact_nodes(nodes) - visible = nodes_visible_to_user(nodes) + # data - An Array of a Hashes mapping an HTML document to nodes to redact. + def redact_document_nodes(all_document_nodes) + all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + visible = nodes_visible_to_user(all_nodes) + metadata = [] - nodes.each do |node| - unless visible.include?(node) + all_document_nodes.each do |entry| + nodes_for_document = entry[:nodes] + doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + metadata << doc_data + + nodes_for_document.each do |node| + next if visible.include?(node) + + doc_data[:visible_reference_count] -= 1 # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text node.replace(text) end end + + metadata end # Returns the nodes visible to the current user. @@ -65,5 +72,11 @@ module Banzai visible end + + def document_nodes(documents) + documents.map do |document| + { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') } + end + end end end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index bf366962aef..b26a41a1f3b 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -2,11 +2,11 @@ module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor def initialize - @texts = [] + @texts_and_contexts = [] end def analyze(text, context = {}) - @texts << Renderer.render(text, context) + @texts_and_contexts << { text: text, context: context } end def references(type, project, current_user = nil) @@ -21,9 +21,10 @@ module Banzai def html_documents # This ensures that we don't memoize anything until we have a number of # text blobs to parse. - return [] if @texts.empty? + return [] if @texts_and_contexts.empty? - @html_documents ||= @texts.map { |html| Nokogiri::HTML.fragment(html) } + @html_documents ||= Renderer.cache_collection_render(@texts_and_contexts) + .map { |html| Nokogiri::HTML.fragment(html) } end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index f306079d833..6c20dec5734 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -9,10 +9,11 @@ module Banzai issues = issues_for_nodes(nodes) - nodes.select do |node| - issue = issue_for_node(issues, node) + readable_issues = Ability. + issues_readable_by_user(issues.values, user).to_set - issue ? can?(user, :read_issue, issue) : false + nodes.select do |node| + readable_issues.include?(issue_for_node(issues, node)) end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 6718acdef7e..a4ae27eefd8 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,5 +1,7 @@ module Banzai module Renderer + extend self + # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -10,11 +12,11 @@ module Banzai # requiring XHTML, such as Atom feeds, need to call `post_process` on the # result, providing the appropriate `pipeline` option. # - # markdown - Markdown String + # text - Markdown String # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def self.render(text, context = {}) + def render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -29,7 +31,59 @@ module Banzai end end - def self.render_result(text, context = {}) + # Perform multiple render from an Array of Markdown String into an + # Array of HTML-safe String of HTML. + # + # As the rendered Markdown String can be already cached read all the data + # from the cache using Rails.cache.read_multi operation. If the Markdown String + # is not in the cache or it's not cacheable (no cache_key entry is provided in + # the context) the Markdown String is rendered and stored in the cache so the + # next render call gets the rendered HTML-safe String from the cache. + # + # For further explanation see #render method comments. + # + # texts_and_contexts - An Array of Hashes that contains the Markdown String (:text) + # an options passed to our HTML Pipeline (:context) + # + # If on the :context you specify a :cache_key entry will be used to retrieve it + # and cache the result of rendering the Markdown String. + # + # Returns an Array containing HTML-safe String instances. + # + # Example: + # texts_and_contexts + # => [{ text: '### Hello', + # context: { cache_key: [note, :note] } }] + def cache_collection_render(texts_and_contexts) + items_collection = texts_and_contexts.each_with_index do |item, index| + context = item[:context] + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + + item[:cache_key] = cache_key if cache_key + end + + cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + + items_in_cache = [] + items_not_in_cache = [] + + unless cacheable_items.empty? + items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) + items_not_in_cache = cacheable_items.reject do |item| + item[:rendered] = items_in_cache[item[:cache_key]] + items_in_cache.key?(item[:cache_key]) + end + end + + (items_not_in_cache + non_cacheable_items).each do |item| + item[:rendered] = render(item[:text], item[:context]) + Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] + end + + items_collection.map { |item| item[:rendered] } + end + + def render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -48,7 +102,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def self.post_process(html, context) + def post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -61,7 +115,7 @@ module Banzai private - def self.cacheless_render(text, context = {}) + def cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -74,9 +128,17 @@ module Banzai end end - def self.full_cache_key(cache_key, pipeline_name) + def full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end + + # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. + # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key + # method. + def full_cache_multi_key(cache_key, pipeline_name) + return unless cache_key + Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + end end end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 17bb99a2ae5..a6b9beecded 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -9,22 +9,14 @@ module Ci end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }, 500) + handle_api_exception(exception) end content_type :txt, 'text/plain' content_type :json, 'application/json' format :json + helpers ::SentryHelper helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 260ac81f5fa..9f3b582a263 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -20,8 +20,13 @@ module Ci build = Ci::RegisterBuildService.new.execute(current_runner) if build + Gitlab::Metrics.add_event(:build_found, + project: build.project.path_with_namespace) + present build, with: Entities::BuildDetails else + Gitlab::Metrics.add_event(:build_not_found) + not_found! end end @@ -42,6 +47,9 @@ module Ci build.update_attributes(trace: params[:trace]) if params[:trace] + Gitlab::Metrics.add_event(:update_build, + project: build.project.path_with_namespace) + case params[:state].to_s when 'success' build.success diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 1d7126a432d..3decc3b1a26 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -1,5 +1,37 @@ module Ci module Charts + module DailyInterval + def grouped_count(query) + query. + group("DATE(#{Ci::Build.table_name}.created_at)"). + count(:created_at). + transform_keys { |date| date.strftime(@format) } + end + + def interval_step + @interval_step ||= 1.day + end + end + + module MonthlyInterval + def grouped_count(query) + if Gitlab::Database.postgresql? + query. + group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')"). + count(:created_at). + transform_keys(&:squish) + else + query. + group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')"). + count(:created_at) + end + end + + def interval_step + @interval_step ||= 1.month + end + end + class Chart attr_reader :labels, :total, :success, :project, :build_times @@ -13,47 +45,59 @@ module Ci collect end - def push(from, to, format) - @labels << from.strftime(format) - @total << project.builds. - where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). - count(:all) - @success << project.builds. - where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). - success.count(:all) + def collect + query = project.builds. + where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from) + + totals_count = grouped_count(query) + success_count = grouped_count(query.success) + + current = @from + while current < @to + label = current.strftime(@format) + + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) + + current += interval_step + end end end class YearChart < Chart - def collect - 13.times do |i| - start_month = (Date.today.years_ago(1) + i.month).beginning_of_month - end_month = start_month.end_of_month + include MonthlyInterval - push(start_month, end_month, "%d %B %Y") - end + def initialize(*) + @to = Date.today.end_of_month + @from = @to.years_ago(1).beginning_of_month + @format = '%d %B %Y' + + super end end class MonthChart < Chart - def collect - 30.times do |i| - start_day = Date.today - 30.days + i.days - end_day = Date.today - 30.days + i.day + 1.day + include DailyInterval - push(start_day, end_day, "%d %B") - end + def initialize(*) + @to = Date.today + @from = @to - 30.days + @format = '%d %B' + + super end end class WeekChart < Chart - def collect - 7.times do |i| - start_day = Date.today - 7.days + i.days - end_day = Date.today - 7.days + i.day + 1.day + include DailyInterval - push(start_day, end_day, "%d %B") - end + def initialize(*) + @to = Date.today + @from = @to - 7.days + @format = '%d %B' + + super end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 01ef13df57a..47efd5bd9f2 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,21 +4,11 @@ module Ci include Gitlab::Ci::Config::Node::LegacyValidationHelpers - DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, - :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables, - :environment] - ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] - ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :path, :cache, :stages def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @config = @ci_config.to_hash - @path = path unless @ci_config.valid? @@ -26,64 +16,42 @@ module Ci end initial_parsing - validate! rescue Gitlab::Ci::Config::Loader::FormatError => e raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - builds.select do |build| - build[:stage] == stage && - process?(build[:only], build[:except], ref, tag, trigger_request) + def jobs_for_ref(ref, tag = false, trigger_request = nil) + @jobs.select do |_, job| + process?(job[:only], job[:except], ref, tag, trigger_request) end end - def builds - @jobs.map do |name, job| - build_job(name, job) + def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).select do |_, job| + job[:stage] == stage end end - def global_variables - @variables - end - - def job_variables(name) - job = @jobs[name.to_sym] - return [] unless job - - job[:variables] || [] + def builds_for_ref(ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).map do |name, _| + build_attributes(name) + end end - private - - def initial_parsing - @before_script = @ci_config.before_script - @image = @ci_config.image - @after_script = @ci_config.after_script - @services = @ci_config.services - @variables = @ci_config.variables - @stages = @ci_config.stages - @cache = @ci_config.cache - - @jobs = {} - - @config.except!(*ALLOWED_YAML_KEYS) - @config.each { |name, param| add_job(name, param) } - - raise ValidationError, "Please define at least one job" if @jobs.none? + def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _| + build_attributes(name) + end end - def add_job(name, job) - return if name.to_s.start_with?('.') - - raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script) - - stage = job[:stage] || job[:type] || DEFAULT_STAGE - @jobs[name] = { stage: stage }.merge(job) + def builds + @jobs.map do |name, _| + build_attributes(name) + end end - def build_job(name, job) + def build_attributes(name) + job = @jobs[name.to_sym] || {} { stage_idx: @stages.index(job[:stage]), stage: job[:stage], @@ -94,12 +62,11 @@ module Ci # - before script should be a concatenated command commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], - name: name, - only: job[:only], - except: job[:except], + name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', environment: job[:environment], + yaml_variables: yaml_variables(name), options: { image: job[:image] || @image, services: job[:services] || @services, @@ -111,154 +78,59 @@ module Ci } end - def validate! - @jobs.each do |name, job| - validate_job!(name, job) - end + private - true - end + def initial_parsing + ## + # Global config + # + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache - def validate_job!(name, job) - validate_job_name!(name) - validate_job_keys!(name, job) - validate_job_types!(name, job) - validate_job_script!(name, job) - - validate_job_stage!(name, job) if job[:stage] - validate_job_variables!(name, job) if job[:variables] - validate_job_cache!(name, job) if job[:cache] - validate_job_artifacts!(name, job) if job[:artifacts] - validate_job_dependencies!(name, job) if job[:dependencies] - end + ## + # Jobs + # + @jobs = @ci_config.jobs - def validate_job_name!(name) - if name.blank? || !validate_string(name) - raise ValidationError, "job name should be non-empty string" - end - end + @jobs.each do |name, job| + # logical validation for job - def validate_job_keys!(name, job) - job.keys.each do |key| - unless ALLOWED_JOB_KEYS.include? key - raise ValidationError, "#{name} job: unknown parameter #{key}" - end + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) end end - def validate_job_types!(name, job) - if job[:image] && !validate_string(job[:image]) - raise ValidationError, "#{name} job: image should be a string" - end + def yaml_variables(name) + variables = (@variables || {}) + .merge(job_variables(name)) - if job[:services] && !validate_array_of_strings(job[:services]) - raise ValidationError, "#{name} job: services should be an array of strings" - end - - if job[:tags] && !validate_array_of_strings(job[:tags]) - raise ValidationError, "#{name} job: tags parameter should be an array of strings" - end - - if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" - end - - if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" - end - - if job[:allow_failure] && !validate_boolean(job[:allow_failure]) - raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" - end - - if job[:when] && !job[:when].in?(%w[on_success on_failure always]) - raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" - end - - if job[:environment] && !validate_environment(job[:environment]) - raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" + variables.map do |key, value| + { key: key, value: value, public: true } end end - def validate_job_script!(name, job) - if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name} job: script should be a string or an array of a strings" - end - - if job[:before_script] && !validate_array_of_strings(job[:before_script]) - raise ValidationError, "#{name} job: before_script should be an array of strings" - end + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job - if job[:after_script] && !validate_array_of_strings(job[:after_script]) - raise ValidationError, "#{name} job: after_script should be an array of strings" - end + job[:variables] || {} end def validate_job_stage!(name, job) + return unless job[:stage] + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" end end - def validate_job_variables!(name, job) - unless validate_variables(job[:variables]) - raise ValidationError, - "#{name} job: variables should be a map of key-value strings" - end - end - - def validate_job_cache!(name, job) - job[:cache].keys.each do |key| - unless ALLOWED_CACHE_KEYS.include? key - raise ValidationError, "#{name} job: cache unknown parameter #{key}" - end - end - - if job[:cache][:key] && !validate_string(job[:cache][:key]) - raise ValidationError, "#{name} job: cache:key parameter should be a string" - end - - if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) - raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" - end - - if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) - raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" - end - end - - def validate_job_artifacts!(name, job) - job[:artifacts].keys.each do |key| - unless ALLOWED_ARTIFACTS_KEYS.include? key - raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" - end - end - - if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) - raise ValidationError, "#{name} job: artifacts:name parameter should be a string" - end - - if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) - raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" - end - - if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) - raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" - end - - if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) - raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" - end - - if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) - raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" - end - end - def validate_job_dependencies!(name, job) - unless validate_array_of_strings(job[:dependencies]) - raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" - end + return unless job[:dependencies] stage_index = @stages.index(job[:stage]) diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb deleted file mode 100644 index bb2bdbed495..00000000000 --- a/lib/ci/static_model.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database. -module Ci - module StaticModel - extend ActiveSupport::Concern - - module ClassMethods - # Used by ActiveRecord's polymorphic association to set object_id - def primary_key - 'id' - end - - # Used by ActiveRecord's polymorphic association to set object_type - def base_class - self - end - end - - # Used by AR for fetching attributes - # - # Pass it along if we respond to it. - def [](key) - send(key) if respond_to?(key) - end - - def to_param - id - end - - def new_record? - false - end - - def persisted? - false - end - - def destroyed? - false - end - - def ==(other) - if other.is_a? ::Ci::StaticModel - id == other.id - else - super - end - end - end -end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 42232b7129d..2edddb84fc3 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,62 +7,91 @@ module ContainerRegistry MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + # Taken from: FaradayMiddleware::FollowRedirects + REDIRECT_CODES = Set.new [301, 302, 303, 307] + def initialize(base_uri, options = {}) @base_uri = base_uri - @faraday = Faraday.new(@base_uri) do |conn| - initialize_connection(conn, options) - end + @options = options end def repository_tags(name) - response_body @faraday.get("/v2/#{name}/tags/list") + response_body faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - response_body @faraday.get("/v2/#{name}/manifests/#{reference}") + response_body faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) - response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response = faraday.head("/v2/#{name}/manifests/#{reference}") response.headers['docker-content-digest'] if response.success? end def delete_repository_tag(name, reference) - @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + faraday.delete("/v2/#{name}/manifests/#{reference}").success? end def blob(name, digest, type = nil) - headers = {} - headers['Accept'] = type if type - response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) + type ||= 'application/octet-stream' + response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true end def delete_blob(name, digest) - @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + faraday.delete("/v2/#{name}/blobs/#{digest}").success? end - + private - + def initialize_connection(conn, options) conn.request :json + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + + def accept_manifest(conn) conn.headers['Accept'] = MANIFEST_VERSION conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + end - if options[:user] && options[:password] - conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) - elsif options[:token] - conn.request(:authorization, :bearer, options[:token].to_s) + def response_body(response, allow_redirect: false) + if allow_redirect && REDIRECT_CODES.include?(response.status) + response = redirect_response(response.headers['location']) end - conn.adapter :net_http + response.body if response && response.success? + end + + def redirect_response(location) + return unless location + + # We explicitly remove authorization token + faraday_blob.get(location) do |req| + req['Authorization'] = '' + end end - def response_body(response) - response.body if response.success? + def faraday + @faraday ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + accept_manifest(conn) + end + end + + def faraday_blob + @faraday_blob ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + end end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 708d01b95a1..59040199920 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -53,7 +53,7 @@ module ContainerRegistry def config return unless config_blob - @config ||= ContainerRegistry::Config.new(self, config_blob) + @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data end def created_at diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 51e46da82cc..a4558d157c0 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -94,7 +94,7 @@ module ExtractsPath @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } @options = HashWithIndifferentAccess.new(@options) - @id = Addressable::URI.unescape(get_id) + @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository if @options[:extended_sha1].blank? @@ -119,6 +119,7 @@ module ExtractsPath private + # overriden in subclasses, do not remove def get_id id = params[:id] || params[:ref] id += "/" + params[:path] unless params[:path].blank? diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 831f1e635ba..a533bac2692 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -7,6 +7,7 @@ module Gitlab module Access class AccessDeniedError < StandardError; end + NO_ACCESS = 0 GUEST = 10 REPORTER = 20 DEVELOPER = 30 @@ -14,9 +15,10 @@ module Gitlab OWNER = 50 # Branch protection settings - PROTECTION_NONE = 0 - PROTECTION_DEV_CAN_PUSH = 1 - PROTECTION_FULL = 2 + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 class << self def values @@ -54,6 +56,7 @@ module Gitlab def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, } diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb deleted file mode 100644 index 04676fdb748..00000000000 --- a/lib/gitlab/akismet_helper.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module AkismetHelper - def akismet_enabled? - current_application_settings.akismet_enabled - end - - def akismet_client - @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, - Gitlab.config.gitlab.url) - end - - def client_ip(env) - env['action_dispatch.remote_ip'].to_s - end - - def user_agent(env) - env['HTTP_USER_AGENT'] - end - - def check_for_spam?(project, user) - akismet_enabled? && !project.team.member?(user) - end - - def is_spam?(environment, user, text) - client = akismet_client - ip_address = client_ip(environment) - user_agent = user_agent(environment) - - params = { - type: 'comment', - text: text, - created_at: DateTime.now, - author: user.name, - author_email: user.email, - referrer: environment['HTTP_REFERER'], - } - - begin - is_spam, is_blatant = client.check(ip_address, user_agent, params) - is_spam || is_blatant - rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") - false - end - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index db1704af75e..91f0270818a 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -10,13 +10,12 @@ module Gitlab if valid_ci_request?(login, password, project) result.type = :ci - elsif result.user = find_with_user_password(login, password) - result.type = :gitlab_or_ldap - elsif result.user = oauth_access_token_check(login, password) - result.type = :oauth + else + result = populate_result(login, password) end - rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) + success = result.user.present? || [:ci, :missing_personal_token].include?(result.type) + rate_limit!(ip, success: success, login: login) result end @@ -76,10 +75,43 @@ module Gitlab end end + def populate_result(login, password) + result = + user_with_password_for_git(login, password) || + oauth_access_token_check(login, password) || + personal_access_token_check(login, password) + + if result + result.type = nil unless result.user + + if result.user && result.user.two_factor_enabled? && result.type == :gitlab_or_ldap + result.type = :missing_personal_token + end + end + + result || Result.new + end + + def user_with_password_for_git(login, password) + user = find_with_user_password(login, password) + Result.new(user, :gitlab_or_ldap) if user + end + def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) + if token && token.accessible? + user = User.find_by(id: token.resource_owner_id) + Result.new(user, :oauth) + end + end + end + + def personal_access_token_check(login, password) + if login && password + user = User.find_by_personal_access_token(password) + validation = User.by_login(login) + Result.new(user, :personal_token) if user == validation end end end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index c94bfc0e65f..39b43ab5489 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -1,24 +1,14 @@ module Gitlab class AwardEmoji CATEGORIES = { - other: "Other", objects: "Objects", - places: "Places", - travel_places: "Travel", - emoticons: "Emoticons", - objects_symbols: "Symbols", + travel: "Travel", + symbols: "Symbols", nature: "Nature", - celebration: "Celebration", people: "People", activity: "Activity", flags: "Flags", - food_drink: "Food" - }.with_indifferent_access - - CATEGORY_ALIASES = { - symbols: "objects_symbols", - foods: "food_drink", - travel: "travel_places" + food: "Food" }.with_indifferent_access def self.normalize_emoji_name(name) @@ -35,7 +25,7 @@ module Gitlab # Skip Fitzpatrick(tone) modifiers next if data["category"] == "modifier" - category = CATEGORY_ALIASES[data["category"]] || data["category"] + category = data["category"] @emoji_by_category[category] << data end @@ -57,9 +47,9 @@ module Gitlab def self.aliases @aliases ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) - JSON.parse(File.read(json_path)) - end + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + JSON.parse(File.read(json_path)) + end end # Returns an Array of Emoji names and their asset URLs. diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb deleted file mode 100644 index 478f145bfed..00000000000 --- a/lib/gitlab/backend/grack_auth.rb +++ /dev/null @@ -1,163 +0,0 @@ -module Grack - class AuthSpawner - def self.call(env) - # Avoid issues with instance variables in Grack::Auth persisting across - # requests by creating a new instance for each request. - Auth.new({}).call(env) - end - end - - class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env - - def call(env) - @env = env - @request = Rack::Request.new(env) - @auth = Request.new(env) - - @ci = false - - # Need this patch due to the rails mount - # Need this if under RELATIVE_URL_ROOT - unless Gitlab.config.gitlab.relative_url_root.empty? - # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') - else - @env['PATH_INFO'] = @request.path - end - - @env['SCRIPT_NAME'] = "" - - auth! - - lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call - return lfs_response unless lfs_response.nil? - - if @user.nil? && !@ci - unauthorized - else - render_not_found - end - end - - private - - def auth! - return unless @auth.provided? - - return bad_request unless @auth.basic? - - # Authentication with username and password - login, password = @auth.credentials - - # Allow authentication for GitLab CI service - # if valid token passed - if ci_request?(login, password) - @ci = true - return - end - - @user = authenticate_user(login, password) - end - - def ci_request?(login, password) - matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) - - if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore - - if underscored_service == 'gitlab_ci' - return project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) - service_method = "#{underscored_service}_service" - service = project.send(service_method) - - return service && service.activated? && service.valid_token?(password) - end - end - - false - end - - def oauth_access_token_check(login, password) - if login == "oauth2" && git_cmd == 'git-upload-pack' && password.present? - token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) - end - end - - def authenticate_user(login, password) - user = Gitlab::Auth.find_with_user_password(login, password) - - unless user - user = oauth_access_token_check(login, password) - end - - # If the user authenticated successfully, we reset the auth failure count - # from Rack::Attack for that IP. A client may attempt to authenticate - # with a username and blank password first, and only after it receives - # a 401 error does it present a password. Resetting the count prevents - # false positives from occurring. - # - # Otherwise, we let Rack::Attack know there was a failed authentication - # attempt from this IP. This information is stored in the Rails cache - # (Redis) and will be used by the Rack::Attack middleware to decide - # whether to block requests from this IP. - config = Gitlab.config.rack_attack.git_basic_auth - - if config.enabled - if user - # A successful login will reset the auth failure count from this IP - Rack::Attack::Allow2Ban.reset(@request.ip, config) - else - banned = Rack::Attack::Allow2Ban.filter(@request.ip, config) do - # Unless the IP is whitelisted, return true so that Allow2Ban - # increments the counter (stored in Rails.cache) for the IP - if config.ip_whitelist.include?(@request.ip) - false - else - true - end - end - - if banned - Rails.logger.info "IP #{@request.ip} failed to login " \ - "as #{login} but has been temporarily banned from Git auth" - end - end - end - - user - end - - def git_cmd - if @request.get? - @request.params['service'] - elsif @request.post? - File.basename(@request.path) - else - nil - end - end - - def project - return @project if defined?(@project) - - @project = project_by_path(@request.path_info) - end - - def project_by_path(path) - if m = /^([\w\.\/-]+)\.git/.match(path).to_a - path_with_namespace = m.last - path_with_namespace.gsub!(/\.wiki$/, '') - - path_with_namespace[0] = '' if path_with_namespace.start_with?('/') - Project.find_with_namespace(path_with_namespace) - end - end - - def render_not_found - [404, { "Content-Type" => "text/plain" }, ["Not Found"]] - end - end -end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 34e0143a82e..839a4fa30d5 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -60,16 +60,18 @@ module Gitlab end # Fork repository to new namespace - # storage - project's storage path + # forked_from_storage - forked-from project's storage path # path - project path with namespace + # forked_to_storage - forked-to project's storage path # fork_namespace - namespace for forked project # # Ex. - # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # - def fork_repository(storage, path, fork_namespace) + def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - storage, "#{path}.git", fork_namespace]) + forked_from_storage, "#{path}.git", forked_to_storage, + fork_namespace]) end # Remove repository from file system diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb new file mode 100644 index 00000000000..909fa24fa90 --- /dev/null +++ b/lib/gitlab/badge/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Badge + class Base + def entity + raise NotImplementedError + end + + def status + raise NotImplementedError + end + + def metadata + raise NotImplementedError + end + + def template + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb deleted file mode 100644 index e5e9fab3f5c..00000000000 --- a/lib/gitlab/badge/build.rb +++ /dev/null @@ -1,46 +0,0 @@ -module Gitlab - module Badge - ## - # Build badge - # - class Build - include Gitlab::Application.routes.url_helpers - include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper - - def initialize(project, ref) - @project, @ref = project, ref - @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) - end - - def type - 'image/svg+xml' - end - - def data - File.read(@image[:path]) - end - - def to_s - @image[:name].sub(/\.svg$/, '') - end - - def to_html - link_to(image_tag(image_url, alt: 'build status'), link_url) - end - - def to_markdown - "[](#{link_url})" - end - - def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) - end - - def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) - end - end - end -end diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb new file mode 100644 index 00000000000..f87a7b7942e --- /dev/null +++ b/lib/gitlab/badge/build/metadata.rb @@ -0,0 +1,28 @@ +module Gitlab + module Badge + module Build + ## + # Class that describes build badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + end + + def title + 'build status' + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb new file mode 100644 index 00000000000..50aa45e5406 --- /dev/null +++ b/lib/gitlab/badge/build/status.rb @@ -0,0 +1,37 @@ +module Gitlab + module Badge + module Build + ## + # Build status badge + # + class Status < Badge::Base + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + + @sha = @project.commit(@ref).try(:sha) + end + + def entity + 'build' + end + + def status + @project.pipelines + .where(sha: @sha, ref: @ref) + .status || 'unknown' + end + + def metadata + @metadata ||= Build::Metadata.new(self) + end + + def template + @template ||= Build::Template.new(self) + end + end + end + end +end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb new file mode 100644 index 00000000000..2b95ddfcb53 --- /dev/null +++ b/lib/gitlab/badge/build/template.rb @@ -0,0 +1,47 @@ +module Gitlab + module Badge + module Build + ## + # Class that represents a build badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + success: '#4c1', + failed: '#e05d44', + running: '#dfb317', + pending: '#dfb317', + canceled: '#9f9f9f', + skipped: '#9f9f9f', + unknown: '#9f9f9f' + } + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + @entity.to_s + end + + def value_text + @status.to_s + end + + def key_width + 38 + end + + def value_width + 54 + end + + def value_color + STATUS_COLOR[@status.to_sym] || STATUS_COLOR[:unknown] + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb new file mode 100644 index 00000000000..53588185622 --- /dev/null +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -0,0 +1,30 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that describes coverage badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + @job = badge.job + end + + def title + 'coverage report' + end + + def image_url + coverage_namespace_project_badges_url(@project.namespace, + @project, @ref, + format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb new file mode 100644 index 00000000000..95d925dc7f3 --- /dev/null +++ b/lib/gitlab/badge/coverage/report.rb @@ -0,0 +1,55 @@ +module Gitlab + module Badge + module Coverage + ## + # Test coverage report badge + # + class Report < Badge::Base + attr_reader :project, :ref, :job + + def initialize(project, ref, job = nil) + @project = project + @ref = ref + @job = job + + @pipeline = @project.pipelines + .latest_successful_for(@ref) + .first + end + + def entity + 'coverage' + end + + def status + @coverage ||= raw_coverage + return unless @coverage + + @coverage.to_i + end + + def metadata + @metadata ||= Coverage::Metadata.new(self) + end + + def template + @template ||= Coverage::Template.new(self) + end + + private + + def raw_coverage + return unless @pipeline + + if @job.blank? + @pipeline.coverage + else + @pipeline.builds + .find_by(name: @job) + .try(:coverage) + end + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb new file mode 100644 index 00000000000..06e0d084e9f --- /dev/null +++ b/lib/gitlab/badge/coverage/template.rb @@ -0,0 +1,52 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that represents a coverage badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + good: '#4c1', + acceptable: '#a3c51c', + medium: '#dfb317', + low: '#e05d44', + unknown: '#9f9f9f' + } + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + @entity.to_s + end + + def value_text + @status ? "#{@status}%" : 'unknown' + end + + def key_width + 62 + end + + def value_width + @status ? 36 : 58 + end + + def value_color + case @status + when 95..100 then STATUS_COLOR[:good] + when 90..95 then STATUS_COLOR[:acceptable] + when 75..90 then STATUS_COLOR[:medium] + when 0..75 then STATUS_COLOR[:low] + else + STATUS_COLOR[:unknown] + end + end + end + end + end +end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb new file mode 100644 index 00000000000..548f85b78bb --- /dev/null +++ b/lib/gitlab/badge/metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module Badge + ## + # Abstract class for badge metadata + # + class Metadata + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(badge) + @badge = badge + end + + def to_html + link_to(image_tag(image_url, alt: title), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def title + raise NotImplementedError + end + + def image_url + raise NotImplementedError + end + + def link_url + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb new file mode 100644 index 00000000000..bfeb0052642 --- /dev/null +++ b/lib/gitlab/badge/template.rb @@ -0,0 +1,49 @@ +module Gitlab + module Badge + ## + # Abstract template class for badges + # + class Template + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + raise NotImplementedError + end + + def value_text + raise NotImplementedError + end + + def key_width + raise NotImplementedError + end + + def value_width + raise NotImplementedError + end + + def value_color + raise NotImplementedError + end + + def key_color + '#555' + end + + def key_text_anchor + key_width / 2 + end + + def value_text_anchor + key_width + (value_width / 2) + end + + def width + key_width + value_width + end + end + end +end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb new file mode 100644 index 00000000000..95308aca95f --- /dev/null +++ b/lib/gitlab/changes_list.rb @@ -0,0 +1,25 @@ +module Gitlab + class ChangesList + include Enumerable + + attr_reader :raw_changes + + def initialize(changes) + @raw_changes = changes.kind_of?(String) ? changes.lines : changes + end + + def each(&block) + changes.each(&block) + end + + def changes + @changes ||= begin + @raw_changes.map do |change| + next if change.blank? + oldrev, newrev, ref = change.strip.split(' ') + { oldrev: oldrev, newrev: newrev, ref: ref } + end.compact + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb new file mode 100644 index 00000000000..4b32eb966aa --- /dev/null +++ b/lib/gitlab/checks/change_access.rb @@ -0,0 +1,78 @@ +module Gitlab + module Checks + class ChangeAccess + attr_reader :user_access, :project + + def initialize(change, user_access:, project:) + @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) + @branch_name = Gitlab::Git.branch_name(@ref) + @user_access = user_access + @project = project + end + + def exec + error = push_checks || tag_checks || protected_branch_checks + + if error + GitAccessStatus.new(false, error) + else + GitAccessStatus.new(true) + end + end + + protected + + def protected_branch_checks + return unless project.protected_branch?(@branch_name) + + if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + return "You are not allowed to force push code to a protected branch on this project." + elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + return "You are not allowed to delete protected branches from this project." + end + + if matching_merge_request? + if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) + return + else + "You are not allowed to merge code into protected branches on this project." + end + else + if user_access.can_push_to_branch?(@branch_name) + return + else + "You are not allowed to push code to protected branches on this project." + end + end + end + + def tag_checks + tag_ref = Gitlab::Git.tag_name(@ref) + + if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) + "You are not allowed to change existing tags on this project." + end + end + + def push_checks + if user_access.cannot_do_action?(:push_code) + "You are not allowed to push code to this project." + end + end + + private + + def protected_tag?(tag_name) + project.repository.tag_exists?(tag_name) + end + + def forced_push? + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def matching_merge_request? + Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? + end + end + end +end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb new file mode 100644 index 00000000000..5fe86553bd0 --- /dev/null +++ b/lib/gitlab/checks/force_push.rb @@ -0,0 +1,17 @@ +module Gitlab + module Checks + class ForcePush + def self.force_push?(project, oldrev, newrev) + return false if project.empty_repo? + + # Created or deleted branch + if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) + false + else + missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) + missed_ref.present? + end + end + end + end +end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb new file mode 100644 index 00000000000..849848515da --- /dev/null +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -0,0 +1,18 @@ +module Gitlab + module Checks + class MatchingMergeRequest + def initialize(newrev, branch_name, project) + @newrev = newrev + @branch_name = branch_name + @project = project + end + + def match? + @project.merge_requests + .with_state(:locked) + .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) + .exists? + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index e6cc1529760..ae82c0db3f1 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -8,7 +8,7 @@ module Gitlab # Temporary delegations that should be removed after refactoring # delegate :before_script, :image, :services, :after_script, :variables, - :stages, :cache, to: :@global + :stages, :cache, :jobs, to: :@global def initialize(config) @config = Loader.new(config).load! diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/node/artifacts.rb new file mode 100644 index 00000000000..844bd2fe998 --- /dev/null +++ b/lib/gitlab/ci/config/node/artifacts.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration of job artifacts. + # + class Artifacts < Entry + include Validatable + include Attributable + + ALLOWED_KEYS = %i[name untracked paths when expire_in] + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :name, type: String + validates :untracked, boolean: true + validates :paths, array_of_strings: true + validates :when, + inclusion: { in: %w[on_success on_failure always], + message: 'should be on_success, on_failure ' \ + 'or always' } + validates :expire_in, duration: true + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/node/attributable.rb new file mode 100644 index 00000000000..221b666f9f6 --- /dev/null +++ b/lib/gitlab/ci/config/node/attributable.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + class Config + module Node + module Attributable + extend ActiveSupport::Concern + + class_methods do + def attributes(*attributes) + attributes.flatten.each do |attribute| + define_method(attribute) do + return unless config.is_a?(Hash) + + config[attribute] + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/node/cache.rb index cdf8ba2e35d..b4bda2841ac 100644 --- a/lib/gitlab/ci/config/node/cache.rb +++ b/lib/gitlab/ci/config/node/cache.rb @@ -8,6 +8,12 @@ module Gitlab class Cache < Entry include Configurable + ALLOWED_KEYS = %i[key untracked paths] + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + node :key, Node::Key, description: 'Cache key used to define a cache affinity.' @@ -16,10 +22,6 @@ module Gitlab node :paths, Node::Paths, description: 'Specify which paths should be cached across builds.' - - validations do - validates :config, allowed_keys: true - end end end end diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/node/commands.rb new file mode 100644 index 00000000000..d7657ae314b --- /dev/null +++ b/lib/gitlab/ci/config/node/commands.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a job script. + # + class Commands < Entry + include Validatable + + validations do + include LegacyValidationHelpers + + validate do + unless string_or_array_of_strings?(config) + errors.add(:config, + 'should be a string or an array of strings') + end + end + + def string_or_array_of_strings?(field) + validate_string(field) || validate_array_of_strings(field) + end + end + + def value + Array(@config) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index 37936fc8242..2de82d40c9d 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -25,10 +25,14 @@ module Gitlab private - def create_node(key, factory) - factory.with(value: @config[key], key: key, parent: self) + def compose! + self.class.nodes.each do |key, factory| + factory + .value(@config[key]) + .with(key: key, parent: self) - factory.create! + @entries[key] = factory.create! + end end class_methods do @@ -36,24 +40,25 @@ module Gitlab Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end - private + private # rubocop:disable Lint/UselessAccessModifier - def node(symbol, entry_class, metadata) - factory = Node::Factory.new(entry_class) + def node(key, node, metadata) + factory = Node::Factory.new(node) .with(description: metadata[:description]) - (@nodes ||= {}).merge!(symbol.to_sym => factory) + (@nodes ||= {}).merge!(key.to_sym => factory) end def helpers(*nodes) nodes.each do |symbol| define_method("#{symbol}_defined?") do - @nodes[symbol].try(:defined?) + @entries[symbol].specified? if @entries[symbol] end define_method("#{symbol}_value") do - raise Entry::InvalidError unless valid? - @nodes[symbol].try(:value) + return unless @entries[symbol] && @entries[symbol].valid? + + @entries[symbol].value end alias_method symbol.to_sym, "#{symbol}_value".to_sym diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 9e79e170a4f..0c782c422b5 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -8,30 +8,31 @@ module Gitlab class Entry class InvalidError < StandardError; end - attr_reader :config + attr_reader :config, :metadata attr_accessor :key, :parent, :description - def initialize(config) + def initialize(config, **metadata) @config = config - @nodes = {} + @metadata = metadata + @entries = {} + @validator = self.class.validator.new(self) - @validator.validate + @validator.validate(:new) end def process! - return if leaf? return unless valid? compose! - process_nodes! + descendants.each(&:process!) end - def nodes - @nodes.values + def leaf? + @entries.none? end - def leaf? - self.class.nodes.none? + def descendants + @entries.values end def ancestors @@ -43,27 +44,30 @@ module Gitlab end def errors - @validator.messages + nodes.flat_map(&:errors) + @validator.messages + descendants.flat_map(&:errors) end def value if leaf? @config else - defined = @nodes.select { |_key, value| value.defined? } - Hash[defined.map { |key, node| [key, node.value] }] + meaningful = @entries.select do |_key, value| + value.specified? && value.relevant? + end + + Hash[meaningful.map { |key, entry| [key, entry.value] }] end end - def defined? + def specified? true end - def self.default + def relevant? + true end - def self.nodes - {} + def self.default end def self.validator @@ -73,17 +77,6 @@ module Gitlab private def compose! - self.class.nodes.each do |key, essence| - @nodes[key] = create_node(key, essence) - end - end - - def process_nodes! - nodes.each(&:process!) - end - - def create_node(key, essence) - raise NotImplementedError end end end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 5919a283283..707b052e6a8 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -10,35 +10,60 @@ module Gitlab def initialize(node) @node = node + @metadata = {} @attributes = {} end + def value(value) + @value = value + self + end + + def metadata(metadata) + @metadata.merge!(metadata) + self + end + def with(attributes) @attributes.merge!(attributes) self end def create! - raise InvalidFactory unless @attributes.has_key?(:value) + raise InvalidFactory unless defined?(@value) - fabricate.tap do |entry| - entry.key = @attributes[:key] - entry.parent = @attributes[:parent] - entry.description = @attributes[:description] + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @value.nil? + Node::Undefined.new( + fabricate_undefined + ) + else + fabricate(@node, @value) end end private - def fabricate + def fabricate_undefined ## - # We assume that unspecified entry is undefined. - # See issue #18775. + # If node has a default value we fabricate concrete node + # with default value. # - if @attributes[:value].nil? - Node::Undefined.new(@node) + if @node.default.nil? + fabricate(Node::Null) else - @node.new(@attributes[:value]) + fabricate(@node, @node.default) + end + end + + def fabricate(node, value = nil) + node.new(value, @metadata).tap do |entry| + entry.key = @attributes[:key] + entry.parent = @attributes[:parent] + entry.description = @attributes[:description] end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index f92e1eccbcf..ccd539fb003 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -34,10 +34,36 @@ module Gitlab description: 'Configure caching between build jobs.' helpers :before_script, :image, :services, :after_script, - :variables, :stages, :types, :cache + :variables, :stages, :types, :cache, :jobs - def stages - stages_defined? ? stages_value : types_value + private + + def compose! + super + + compose_jobs! + compose_deprecated_entries! + end + + def compose_jobs! + factory = Node::Factory.new(Node::Jobs) + .value(@config.except(*self.class.nodes.keys)) + .with(key: :jobs, parent: self, + description: 'Jobs definition for this pipeline') + + @entries[:jobs] = factory.create! + end + + def compose_deprecated_entries! + ## + # Deprecated `:types` key workaround - if types are defined and + # stages are not defined we use types definition as stages. + # + if types_defined? && !stages_defined? + @entries[:stages] = @entries[:types] + end + + @entries.delete(:types) end end end diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/node/hidden_job.rb new file mode 100644 index 00000000000..073044b66f8 --- /dev/null +++ b/lib/gitlab/ci/config/node/hidden_job.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a hidden CI/CD job. + # + class HiddenJob < Entry + include Validatable + + validations do + validates :config, type: Hash + validates :config, presence: true + end + + def relevant? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb new file mode 100644 index 00000000000..e84737acbb9 --- /dev/null +++ b/lib/gitlab/ci/config/node/job.rb @@ -0,0 +1,123 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a concrete CI/CD job. + # + class Job < Entry + include Configurable + include Attributable + + ALLOWED_KEYS = %i[tags script only except type image services allow_failure + type stage when artifacts cache dependencies before_script + after_script variables environment] + + attributes :tags, :allow_failure, :when, :environment, :dependencies + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :config, presence: true + validates :name, presence: true + validates :name, type: Symbol + + with_options allow_nil: true do + validates :tags, array_of_strings: true + validates :allow_failure, boolean: true + validates :when, + inclusion: { in: %w[on_success on_failure always manual], + message: 'should be on_success, on_failure, ' \ + 'always or manual' } + validates :environment, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message } + validates :environment, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + validates :dependencies, array_of_strings: true + end + end + + node :before_script, Script, + description: 'Global before script overridden in this job.' + + node :script, Commands, + description: 'Commands that will be executed in this job.' + + node :stage, Stage, + description: 'Pipeline stage this job will be executed into.' + + node :type, Stage, + description: 'Deprecated: stage this job will be executed into.' + + node :after_script, Script, + description: 'Commands that will be executed when finishing job.' + + node :cache, Cache, + description: 'Cache definition for this job.' + + node :image, Image, + description: 'Image that will be used to execute this job.' + + node :services, Services, + description: 'Services that will be used to execute this job.' + + node :only, Trigger, + description: 'Refs policy this job will be executed for.' + + node :except, Trigger, + description: 'Refs policy this job will be executed for.' + + node :variables, Variables, + description: 'Environment variables available for this job.' + + node :artifacts, Artifacts, + description: 'Artifacts configuration for this job.' + + helpers :before_script, :script, :stage, :type, :after_script, + :cache, :image, :services, :only, :except, :variables, + :artifacts + + def name + @metadata[:name] + end + + def value + @config.merge(to_hash.compact) + end + + private + + def to_hash + { name: name, + before_script: before_script, + script: script, + image: image, + services: services, + stage: stage, + cache: cache, + only: only, + except: except, + variables: variables_defined? ? variables : nil, + artifacts: artifacts, + after_script: after_script } + end + + def compose! + super + + if type_defined? && !stage_defined? + @entries[:stage] = @entries[:type] + end + + @entries.delete(:type) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb new file mode 100644 index 00000000000..51683c82ceb --- /dev/null +++ b/lib/gitlab/ci/config/node/jobs.rb @@ -0,0 +1,48 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a set of jobs. + # + class Jobs < Entry + include Validatable + + validations do + validates :config, type: Hash + + validate do + unless has_visible_job? + errors.add(:config, 'should contain at least one visible job') + end + end + + def has_visible_job? + config.any? { |name, _| !hidden?(name) } + end + end + + def hidden?(name) + name.to_s.start_with?('.') + end + + private + + def compose! + @config.each do |name, config| + node = hidden?(name) ? Node::HiddenJob : Node::Job + + factory = Node::Factory.new(node) + .value(config || {}) + .metadata(name: name) + .with(key: name, parent: self, + description: "#{name} job definition.") + + @entries[name] = factory.create! + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 4d9a508796a..0c291efe6a5 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -41,10 +41,6 @@ module Gitlab false end - def validate_environment(value) - value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex - end - def validate_boolean(value) value.in?([true, false]) end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb new file mode 100644 index 00000000000..88a5f53f13c --- /dev/null +++ b/lib/gitlab/ci/config/node/null.rb @@ -0,0 +1,34 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an undefined node. + # + # Implements the Null Object pattern. + # + class Null < Entry + def value + nil + end + + def valid? + true + end + + def errors + [] + end + + def specified? + false + end + + def relevant? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/node/stage.rb new file mode 100644 index 00000000000..cbc97641f5a --- /dev/null +++ b/lib/gitlab/ci/config/node/stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a stage for a job. + # + class Stage < Entry + include Validatable + + validations do + validates :config, type: String + end + + def self.default + 'test' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/node/trigger.rb new file mode 100644 index 00000000000..d8b31975088 --- /dev/null +++ b/lib/gitlab/ci/config/node/trigger.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a trigger policy for the job. + # + class Trigger < Entry + include Validatable + + validations do + include LegacyValidationHelpers + + validate :array_of_strings_or_regexps + + def array_of_strings_or_regexps + unless validate_array_of_strings_or_regexps(config) + errors.add(:config, 'should be an array of strings or regexps') + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb index 699605e1e3a..45fef8c3ae5 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -3,24 +3,13 @@ module Gitlab class Config module Node ## - # This class represents an undefined entry node. + # This class represents an unspecified entry node. # - # It takes original entry class as configuration and returns default - # value of original entry as self value. + # It decorates original entry adding method that indicates it is + # unspecified. # - # - class Undefined < Entry - include Validatable - - validations do - validates :config, type: Class - end - - def value - @config.default - end - - def defined? + class Undefined < SimpleDelegator + def specified? false end end diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb index f6e2896dfb2..085e6e988d1 100644 --- a/lib/gitlab/ci/config/node/validatable.rb +++ b/lib/gitlab/ci/config/node/validatable.rb @@ -7,13 +7,11 @@ module Gitlab class_methods do def validator - validator = Class.new(Node::Validator) - - if defined?(@validations) - @validations.each { |rules| validator.class_eval(&rules) } + @validator ||= Class.new(Node::Validator).tap do |validator| + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end end - - validator end private diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb index 758a6cf4356..43c7e102b50 100644 --- a/lib/gitlab/ci/config/node/validator.rb +++ b/lib/gitlab/ci/config/node/validator.rb @@ -21,18 +21,19 @@ module Gitlab 'Validator' end - def unknown_keys - return [] unless config.is_a?(Hash) - - config.keys - @node.class.nodes.keys - end - private def location predecessors = ancestors.map(&:key).compact - current = key || @node.class.name.demodulize.underscore - predecessors.append(current).join(':') + predecessors.append(key_name).join(':') + end + + def key_name + if key.blank? + @node.class.name.demodulize.underscore.humanize + else + key + end end end end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb index 7b2f57990b5..e20908ad3cb 100644 --- a/lib/gitlab/ci/config/node/validators.rb +++ b/lib/gitlab/ci/config/node/validators.rb @@ -5,10 +5,11 @@ module Gitlab module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - if record.unknown_keys.any? - unknown_list = record.unknown_keys.join(', ') - record.errors.add(:config, - "contains unknown keys: #{unknown_list}") + unknown_keys = record.config.try(:keys).to_a - options[:in] + + if unknown_keys.any? + record.errors.add(:config, 'contains unknown keys: ' + + unknown_keys.join(', ')) end end end @@ -33,6 +34,16 @@ module Gitlab end end + class DurationValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_duration(value) + record.errors.add(attribute, 'should be a duration') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -49,7 +60,8 @@ module Gitlab raise unless type.is_a?(Class) unless value.is_a?(type) - record.errors.add(attribute, "should be a #{type.name}") + message = options[:message] || "should be a #{type.name}" + record.errors.add(attribute, message) end end end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 9bef9037ad6..58f86abc5c4 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -22,7 +22,9 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) - @extractor.issues + @extractor.issues.reject do |issue| + @extractor.project.forked_from?(issue.project) # Don't extract issues on original project + end end end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb new file mode 100644 index 00000000000..dff9e29c6a5 --- /dev/null +++ b/lib/gitlab/conflict/file.rb @@ -0,0 +1,197 @@ +module Gitlab + module Conflict + class File + include Gitlab::Routing.url_helpers + include IconsHelper + + class MissingResolution < StandardError + end + + CONTEXT_LINES = 3 + + attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository + + def initialize(merge_file_result, conflict, merge_request:) + @merge_file_result = merge_file_result + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @merge_request = merge_request + @repository = merge_request.project.repository + @match_line_headers = {} + end + + # Array of Gitlab::Diff::Line objects + def lines + @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data], + our_path: our_path, + their_path: their_path, + parent_file: self) + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line.type + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line.type == 'new' + when 'origin' + next unless line.type == 'old' + else + raise MissingResolution, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def highlight_lines! + their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + + their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + + lines.each do |line| + if line.type == 'old' + line.rich_text = their_highlight[line.old_line - 1].try(:html_safe) + else + line.rich_text = our_highlight[line.new_line - 1].try(:html_safe) + end + end + end + + def sections + return @sections if @sections + + chunked_lines = lines.chunk { |line| line.type.nil? }.to_a + match_line = nil + + sections_count = chunked_lines.size + + @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i| + section = nil + + # We need to reduce context sections to CONTEXT_LINES. Conflict sections are + # always shown in full. + if no_conflict + conflict_before = i > 0 + conflict_after = (sections_count - i) > 1 + + if conflict_before && conflict_after + # Create a gap in a long context section. + if lines.length > CONTEXT_LINES * 2 + head_lines = lines.first(CONTEXT_LINES) + tail_lines = lines.last(CONTEXT_LINES) + + # Ensure any existing match line has text for all lines up to the last + # line of its context. + update_match_line_text(match_line, head_lines.last) + + # Insert a new match line after the created gap. + match_line = create_match_line(tail_lines.first) + + section = [ + { conflict: false, lines: head_lines }, + { conflict: false, lines: tail_lines.unshift(match_line) } + ] + end + elsif conflict_after + tail_lines = lines.last(CONTEXT_LINES) + + # Create a gap and insert a match line at the start. + if lines.length > tail_lines.length + match_line = create_match_line(tail_lines.first) + + tail_lines.unshift(match_line) + end + + lines = tail_lines + elsif conflict_before + # We're at the end of the file (no conflicts after), so just remove extra + # trailing lines. + lines = lines.first(CONTEXT_LINES) + end + end + + # We want to update the match line's text every time unless we've already + # created a gap and its corresponding match line. + update_match_line_text(match_line, lines.last) unless section + + section ||= { conflict: !no_conflict, lines: lines } + section[:id] = line_code(lines.first) unless no_conflict + section + end + end + + def line_code(line) + Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + end + + def create_match_line(line) + Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) + end + + # Any line beginning with a letter, an underscore, or a dollar can be used in a + # match line header. Only context sections can contain match lines, as match lines + # have to exist in both versions of the file. + def find_match_line_header(index) + return @match_line_headers[index] if @match_line_headers.key?(index) + + @match_line_headers[index] = begin + if index >= 0 + line = lines[index] + + if line.type.nil? && line.text.match(/\A[A-Za-z$_]/) + " #{line.text}" + else + find_match_line_header(index - 1) + end + end + end + end + + # Set the match line's text for the current line. A match line takes its start + # position and context header (where present) from itself, and its end position from + # the line passed in. + def update_match_line_text(match_line, line) + return unless match_line + + header = find_match_line_header(match_line.index - 1) + + match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" + end + + def as_json(opts = nil) + { + old_path: their_path, + new_path: our_path, + blob_icon: file_type_icon_class('file', our_mode, our_path), + blob_path: namespace_project_blob_path(merge_request.project.namespace, + merge_request.project, + ::File.join(merge_request.diff_refs.head_sha, our_path)), + sections: sections + } + end + + # Don't try to print merge_request or repository. + def inspect + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode].map do |instance_variable| + value = instance_variable_get("@#{instance_variable}") + + "#{instance_variable}=\"#{value}\"" + end + + "#<#{self.class} #{instance_variables.join(' ')}>" + end + end + end +end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb new file mode 100644 index 00000000000..bbd0427a2c8 --- /dev/null +++ b/lib/gitlab/conflict/file_collection.rb @@ -0,0 +1,57 @@ +module Gitlab + module Conflict + class FileCollection + class ConflictSideMissing < StandardError + end + + attr_reader :merge_request, :our_commit, :their_commit + + def initialize(merge_request) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + end + + def repository + merge_request.project.repository + end + + def merge_index + @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + end + + def files + @files ||= merge_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), + conflict, + merge_request: merge_request) + end + end + + def as_json(opts = nil) + { + target_branch: merge_request.target_branch, + source_branch: merge_request.source_branch, + commit_sha: merge_request.diff_head_sha, + commit_message: default_commit_message, + files: files + } + end + + def default_commit_message + conflict_filenames = merge_index.conflicts.map do |conflict| + "# #{conflict[:ours][:path]}" + end + + <<EOM.chomp +Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}' + +# Conflicts: +#{conflict_filenames.join("\n")} +EOM + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb new file mode 100644 index 00000000000..2d4d55daeeb --- /dev/null +++ b/lib/gitlab/conflict/parser.rb @@ -0,0 +1,71 @@ +module Gitlab + module Conflict + class Parser + class ParserError < StandardError + end + + class UnexpectedDelimiter < ParserError + end + + class MissingEndDelimiter < ParserError + end + + class UnmergeableFile < ParserError + end + + class UnsupportedEncoding < ParserError + end + + def parse(text, our_path:, their_path:, parent_file: nil) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 102400 + + begin + text.to_json + rescue Encoding::UndefinedConversionError + raise UnsupportedEncoding + end + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + raise UnexpectedDelimiter unless type.nil? + + type = 'new' + elsif full_line == conflict_middle + raise UnexpectedDelimiter unless type == 'new' + + type = 'old' + elsif full_line == conflict_end + raise UnexpectedDelimiter unless type == 'old' + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + else + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9dc2602867e..bd681f03173 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -23,7 +23,6 @@ module Gitlab dates.each do |date| date_id = date.to_time.to_i.to_s - @timestamps[date_id] = 0 day_events = events.find { |day_events| day_events["date"] == date } if day_events diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index ffc1814b29d..12fbb78c53e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -30,6 +30,7 @@ module Gitlab signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], gravatar_enabled: Settings.gravatar['enabled'], + koding_enabled: false, sign_in_text: nil, after_sign_up_text: nil, help_page_text: nil, @@ -39,8 +40,8 @@ module Gitlab session_expire_delay: Settings.gitlab['session_expire_delay'], default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], + domain_whitelist: Settings.gitlab['domain_whitelist'], + import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/data_builder/build.rb index 9f45aefda0f..6548e6475c6 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/data_builder/build.rb @@ -1,6 +1,8 @@ module Gitlab - class BuildDataBuilder - class << self + module DataBuilder + module Build + extend self + def build(build) project = build.project commit = build.pipeline diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/data_builder/note.rb index 8bdc89a7751..50fea1232af 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/data_builder/note.rb @@ -1,6 +1,8 @@ module Gitlab - class NoteDataBuilder - class << self + module DataBuilder + module Note + extend self + # Produce a hash of post-receive data # # For all notes: diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb new file mode 100644 index 00000000000..06a783ebc1c --- /dev/null +++ b/lib/gitlab/data_builder/pipeline.rb @@ -0,0 +1,62 @@ +module Gitlab + module DataBuilder + module Pipeline + extend self + + def build(pipeline) + { + object_kind: 'pipeline', + object_attributes: hook_attrs(pipeline), + user: pipeline.user.try(:hook_attrs), + project: pipeline.project.hook_attrs(backward: false), + commit: pipeline.commit.try(:hook_attrs), + builds: pipeline.builds.map(&method(:build_hook_attrs)) + } + end + + def hook_attrs(pipeline) + { + id: pipeline.id, + ref: pipeline.ref, + tag: pipeline.tag, + sha: pipeline.sha, + before_sha: pipeline.before_sha, + status: pipeline.status, + stages: pipeline.stages, + created_at: pipeline.created_at, + finished_at: pipeline.finished_at, + duration: pipeline.duration + } + end + + def build_hook_attrs(build) + { + id: build.id, + stage: build.stage, + name: build.name, + status: build.status, + created_at: build.created_at, + started_at: build.started_at, + finished_at: build.finished_at, + when: build.when, + manual: build.manual?, + user: build.user.try(:hook_attrs), + runner: build.runner && runner_hook_attrs(build.runner), + artifacts_file: { + filename: build.artifacts_file.filename, + size: build.artifacts_size + } + } + end + + def runner_hook_attrs(runner) + { + id: runner.id, + description: runner.description, + active: runner.active?, + is_shared: runner.is_shared? + } + end + end + end +end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/data_builder/push.rb index c8f12577112..4f81863da35 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/data_builder/push.rb @@ -1,6 +1,8 @@ module Gitlab - class PushDataBuilder - class << self + module DataBuilder + module Push + extend self + # Produce a hash of post-receive data # # data = { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 078609c86f1..55b8f888d53 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -55,12 +55,12 @@ module Gitlab end end - private - def self.connection ActiveRecord::Base.connection end + private_class_method :connection + def self.database_version row = connection.execute("SELECT VERSION()").first @@ -70,5 +70,7 @@ module Gitlab row.first end end + + private_class_method :database_version end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dec20d8659b..927f9dad20b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -20,11 +20,19 @@ module Gitlab if Database.postgresql? options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout end add_index(table_name, column_name, options) end + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. (PostgreSQL only) + def disable_statement_timeout + ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -133,6 +141,8 @@ module Gitlab 'in the body of your migration class' end + disable_statement_timeout + transaction do add_column(table, column, type, default: nil) diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index b0c50edba59..e47df508ca2 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -5,7 +5,7 @@ module Gitlab delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, :a_mode, :b_mode, - :submodule?, :too_large?, to: :diff, prefix: false + :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false def initialize(diff, repository:, diff_refs: nil) @diff = diff @@ -63,15 +63,18 @@ module Gitlab diff_refs.try(:head_sha) end + attr_writer :highlighted_diff_lines + # Array of Gitlab::Diff::Line objects def diff_lines - @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a end def highlighted_diff_lines @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end + # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted def parallel_diff_lines @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb new file mode 100644 index 00000000000..2b9fc65b985 --- /dev/null +++ b/lib/gitlab/diff/file_collection/base.rb @@ -0,0 +1,35 @@ +module Gitlab + module Diff + module FileCollection + class Base + attr_reader :project, :diff_options, :diff_view, :diff_refs + + delegate :count, :size, :real_size, to: :diff_files + + def self.default_options + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) + end + + def initialize(diffable, project:, diff_options: nil, diff_refs: nil) + diff_options = self.class.default_options.merge(diff_options || {}) + + @diffable = diffable + @diffs = diffable.raw_diffs(diff_options) + @project = project + @diff_options = diff_options + @diff_refs = diff_refs + end + + def diff_files + @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } + end + + private + + def decorate_diff!(diff) + Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb new file mode 100644 index 00000000000..4dc297ec036 --- /dev/null +++ b/lib/gitlab/diff/file_collection/commit.rb @@ -0,0 +1,14 @@ +module Gitlab + module Diff + module FileCollection + class Commit < Base + def initialize(commit, diff_options:) + super(commit, + project: commit.project, + diff_options: diff_options, + diff_refs: commit.diff_refs) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb new file mode 100644 index 00000000000..20d8f891cc3 --- /dev/null +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -0,0 +1,14 @@ +module Gitlab + module Diff + module FileCollection + class Compare < Base + def initialize(compare, project:, diff_options:, diff_refs: nil) + super(compare, + project: project, + diff_options: diff_options, + diff_refs: diff_refs) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb new file mode 100644 index 00000000000..36348b33943 --- /dev/null +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -0,0 +1,73 @@ +module Gitlab + module Diff + module FileCollection + class MergeRequestDiff < Base + def initialize(merge_request_diff, diff_options:) + @merge_request_diff = merge_request_diff + + super(merge_request_diff, + project: merge_request_diff.project, + diff_options: diff_options, + diff_refs: merge_request_diff.diff_refs) + end + + def diff_files + super.tap { |_| store_highlight_cache } + end + + private + + # Extracted method to highlight in the same iteration to the diff_collection. + def decorate_diff!(diff) + diff_file = super + cache_highlight!(diff_file) if cacheable? + diff_file + end + + def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) + diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + + # + # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted) + # for the highlighted ones, so we just skip their execution. + # If the highlighted diff files lines are not cached we calculate and cache them. + # + # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of + # hashes that represent serialized diff lines. + # + def cache_highlight!(diff_file) + file_path = diff_file.file_path + + if highlight_cache[file_path] + highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path]) + else + highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + end + + def highlight_cache + return @highlight_cache if defined?(@highlight_cache) + + @highlight_cache = Rails.cache.read(cache_key) || {} + @highlight_cache_was_empty = @highlight_cache.empty? + @highlight_cache + end + + def store_highlight_cache + Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty + end + + def cacheable? + @merge_request_diff.present? + end + + def cache_key + [@merge_request_diff, 'highlighted-diff-files', diff_options] + end + end + end + end +end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 649a265a02c..9ea976e18fa 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -40,8 +40,6 @@ module Gitlab def highlight_line(diff_line) return unless diff_file && diff_file.diff_refs - line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' - rich_line = if diff_line.unchanged? || diff_line.added? new_lines[diff_line.new_pos - 1] @@ -51,7 +49,10 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. - "#{line_prefix}#{rich_line}".html_safe if rich_line + if rich_line + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + "#{line_prefix}#{rich_line}".html_safe + end end def inline_diffs diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 789c14518b0..55708d42161 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,27 +1,23 @@ module Gitlab module Diff class InlineDiff - attr_accessor :old_line, :new_line, :offset - - def self.for_lines(lines) - local_edit_indexes = self.find_local_edits(lines) - - inline_diffs = [] + # Regex to find a run of deleted lines followed by the same number of added lines + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 - old_line = lines[old_index] - new_line = lines[new_index] - - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs - - inline_diffs[old_index] = old_diffs - inline_diffs[new_index] = new_diffs - end - - inline_diffs - end + attr_accessor :old_line, :new_line, :offset def initialize(old_line, new_line, offset: 0) @old_line = old_line[offset..-1] @@ -49,22 +45,54 @@ module Gitlab [old_diffs, new_diffs] end - private + class << self + def for_lines(lines) + changed_line_pairs = find_changed_line_pairs(lines) + + inline_diffs = [] - def self.find_local_edits(lines) - line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - joined_line_prefixes = " #{line_prefixes.join} " + changed_line_pairs.each do |old_index, new_index| + old_line = lines[old_index] + new_line = lines[new_index] - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + + inline_diffs[old_index] = old_diffs + inline_diffs[new_index] = new_diffs + end + + inline_diffs end - local_edit_indexes + private + + # Finds pairs of old/new line pairs that represent the same line that changed + def find_changed_line_pairs(lines) + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + + changed_line_pairs = [] + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + changed_line_pairs << [i, i + changed_line_count] + end + end + + changed_line_pairs + end end + private + def longest_common_prefix(a, b) max_length = [a.length, b.length].max diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index c6189d660c2..80a146b4a5a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,11 +2,27 @@ module Gitlab module Diff class Line attr_reader :type, :index, :old_pos, :new_pos + attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos + @parent_file = parent_file + end + + def self.init_from_hash(hash) + new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos]) + end + + def serialize_keys + @serialize_keys ||= %i(text type index old_pos new_pos) + end + + def to_hash + hash = {} + serialize_keys.each { |key| hash[key] = send(key) } + hash end def old_line @@ -29,9 +45,25 @@ module Gitlab type == 'old' end + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text + end + def meta? type == 'match' || type == 'nonewline' end + + def as_json(opts = nil) + { + type: type, + old_line: old_line, + new_line: new_line, + text: text, + rich_text: rich_text || text + } + end end end end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 1c1fc148123..481536a380b 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -8,114 +8,52 @@ module Gitlab end def parallelize - lines = [] - skip_next = false + i = 0 + free_right_index = nil + lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - full_line = line.text - type = line.type - line_code = diff_file.line_code(line) - line_new = line.new_pos - line_old = line.old_pos - position = diff_file.position(line) - - next_line = diff_file.next_line(line.index) - - if next_line - next_line = highlighted_diff_lines[next_line.index] - full_next_line = next_line.text - next_line_code = diff_file.line_code(next_line) - next_type = next_line.type - next_position = diff_file.position(next_line) - end - - case type - when 'match', nil + if line.meta? || line.unchanged? # line in the right panel is the same as in the left one lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - position: position - }, - right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code, - position: position - } + left: line, + right: line } - when 'old' - case next_type - when 'new' - # Left side has text removed, right side has text added - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - position: position - }, - right: { - type: next_type, - number: line_new, - text: full_next_line, - line_code: next_line_code, - position: next_position, - } - } - skip_next = true - when 'old', 'nonewline', nil - # Left side has text removed, right side doesn't have any change - # No next line code, no new line number, no new line text - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - position: position - }, - right: { - type: next_type, - number: nil, - text: "", - line_code: nil, - position: nil - } - } - end - when 'new' - if skip_next - # Change has been already included in previous line so no need to do it again - skip_next = false - next + + free_right_index = nil + i += 1 + elsif line.removed? + lines << { + left: line, + right: nil + } + + # Once we come upon a new line it can be put on the right of this old line + free_right_index ||= i + i += 1 + elsif line.added? + if free_right_index + # If an old line came before this without a line on the right, this + # line can be put to the right of it. + lines[free_right_index][:right] = line + + # If there are any other old lines on the left that don't yet have + # a new counterpart on the right, update the free_right_index + next_free_right_index = free_right_index + 1 + free_right_index = next_free_right_index < i ? next_free_right_index : nil else - # Change is only on the right side, left side has no change lines << { - left: { - type: nil, - number: nil, - text: "", - line_code: line_code, - position: position - }, - right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code, - position: position - } + left: nil, + right: line } + + free_right_index = nil + i += 1 end end end + lines end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 989fff8918e..ecf62dead35 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -73,8 +73,8 @@ module Gitlab diff_refs.complete? end - def to_json - JSON.generate(self.to_h) + def to_json(opts = nil) + JSON.generate(self.to_h, opts) end def type @@ -139,13 +139,19 @@ module Gitlab private def find_diff_file(repository) - diffs = Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ).diffs(paths: paths) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + compare = Gitlab::Git::Commit.find(repository.raw_repository, head_sha) + else + compare = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ) + end + + diff = compare.diffs(paths: paths).first - diff = diffs.first return unless diff Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb new file mode 100644 index 00000000000..ab9537ed7d7 --- /dev/null +++ b/lib/gitlab/downtime_check.rb @@ -0,0 +1,71 @@ +module Gitlab + # Checks if a set of migrations requires downtime or not. + class DowntimeCheck + # The constant containing the boolean that indicates if downtime is needed + # or not. + DOWNTIME_CONST = :DOWNTIME + + # The constant that specifies the reason for the migration requiring + # downtime. + DOWNTIME_REASON_CONST = :DOWNTIME_REASON + + # Checks the given migration paths and returns an Array of + # `Gitlab::DowntimeCheck::Message` instances. + # + # migrations - The migration file paths to check. + def check(migrations) + migrations.map do |path| + require(path) + + migration_class = class_for_migration_file(path) + + unless migration_class.const_defined?(DOWNTIME_CONST) + raise "The migration in #{path} does not specify if it requires " \ + "downtime or not" + end + + if online?(migration_class) + Message.new(path) + else + reason = downtime_reason(migration_class) + + unless reason + raise "The migration in #{path} requires downtime but no reason " \ + "was given" + end + + Message.new(path, true, reason) + end + end + end + + # Checks the given migrations and prints the results to STDOUT/STDERR. + # + # migrations - The migration file paths to check. + def check_and_print(migrations) + check(migrations).each do |message| + puts message.to_s # rubocop: disable Rails/Output + end + end + + # Returns the class for the given migration file path. + def class_for_migration_file(path) + File.basename(path, File.extname(path)).split('_', 2).last.camelize. + constantize + end + + # Returns true if the given migration can be performed without downtime. + def online?(migration) + migration.const_get(DOWNTIME_CONST) == false + end + + # Returns the downtime reason, or nil if none was defined. + def downtime_reason(migration) + if migration.const_defined?(DOWNTIME_REASON_CONST) + migration.const_get(DOWNTIME_REASON_CONST) + else + nil + end + end + end +end diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb new file mode 100644 index 00000000000..40a4815a9a0 --- /dev/null +++ b/lib/gitlab/downtime_check/message.rb @@ -0,0 +1,39 @@ +module Gitlab + class DowntimeCheck + class Message + attr_reader :path, :offline + + OFFLINE = "\e[31moffline\e[0m" + ONLINE = "\e[32monline\e[0m" + + # path - The file path of the migration. + # offline - When set to `true` the migration will require downtime. + # reason - The reason as to why the migration requires downtime. + def initialize(path, offline = false, reason = nil) + @path = path + @offline = offline + @reason = reason + end + + def to_s + label = offline ? OFFLINE : ONLINE + + message = "[#{label}]: #{path}" + + if reason? + message += ":\n\n#{reason}\n\n" + end + + message + end + + def reason? + @reason.present? + end + + def reason + @reason.strip.lines.map(&:strip).join("\n") + end + end + end +end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb new file mode 100644 index 00000000000..5cf9d5ebe28 --- /dev/null +++ b/lib/gitlab/email/handler.rb @@ -0,0 +1,18 @@ +require 'gitlab/email/handler/create_note_handler' +require 'gitlab/email/handler/create_issue_handler' + +module Gitlab + module Email + module Handler + # The `CreateIssueHandler` feature is disabled for the time being. + HANDLERS = [CreateNoteHandler] + + def self.for(mail, mail_key) + HANDLERS.find do |klass| + handler = klass.new(mail, mail_key) + break handler if handler.can_handle? + end + end + end + end +end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb new file mode 100644 index 00000000000..7cccf465334 --- /dev/null +++ b/lib/gitlab/email/handler/base_handler.rb @@ -0,0 +1,61 @@ +module Gitlab + module Email + module Handler + class BaseHandler + attr_reader :mail, :mail_key + + def initialize(mail, mail_key) + @mail = mail + @mail_key = mail_key + end + + def message + @message ||= process_message + end + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + private + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb new file mode 100644 index 00000000000..4e6566af8ab --- /dev/null +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -0,0 +1,52 @@ + +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class CreateIssueHandler < BaseHandler + attr_reader :project_path, :authentication_token + + def initialize(mail, mail_key) + super(mail, mail_key) + @project_path, @authentication_token = + mail_key && mail_key.split('+', 2) + end + + def can_handle? + !authentication_token.nil? + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_issue) + + verify_record!( + record: create_issue, + invalid_exception: InvalidIssueError, + record_name: 'issue') + end + + def author + @author ||= User.find_by(authentication_token: authentication_token) + end + + def project + @project ||= Project.find_with_namespace(project_path) + end + + private + + def create_issue + Issues::CreateService.new( + project, + author, + title: mail.subject, + description: message + ).execute + end + end + end + end +end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb new file mode 100644 index 00000000000..06dae31cc27 --- /dev/null +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -0,0 +1,55 @@ + +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class CreateNoteHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/ + + validate_permission!(:create_note) + + raise NoteableNotFoundError unless sent_notification.noteable + raise EmptyEmailError if message.blank? + + verify_record!( + record: create_note, + invalid_exception: InvalidNoteError, + record_name: 'comment') + end + + def author + sent_notification.recipient + end + + def project + sent_notification.project + end + + def sent_notification + @sent_notification ||= SentNotification.for(mail_key) + end + + private + + def create_note + Notes::CreateService.new( + project, + author, + note: message, + noteable_type: sent_notification.noteable_type, + noteable_id: sent_notification.noteable_id, + commit_id: sent_notification.commit_id, + line_code: sent_notification.line_code + ).execute + end + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 97701b0cd42..0e3b65fceb4 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -35,21 +35,22 @@ module Gitlab def commits return unless compare - @commits ||= Commit.decorate(compare.commits, project) + @commits ||= compare.commits end def diffs return unless compare - - @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository) + + # This diff is more moderated in number of files and lines + @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files end def diffs_count - diffs.count if diffs + diffs.size if diffs end def compare - @opts[:compare] + @opts[:compare] if @opts[:compare] end def diff_refs @@ -97,16 +98,18 @@ module Gitlab if commits.length > 1 namespace_project_compare_url(project_namespace, project, - from: Commit.new(compare.base, project), - to: Commit.new(compare.head, project)) + from: compare.start_commit, + to: compare.head_commit) else namespace_project_commit_url(project_namespace, - project, commits.first) + project, + commits.first) end else unless @action == :delete namespace_project_tree_url(project_namespace, - project, ref_name) + project, + ref_name) end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 1c671a7487b..a40c44eb1bc 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,18 +1,24 @@ + +require_dependency 'gitlab/email/handler' + # Inspired in great part by Discourse's Email::Receiver module Gitlab module Email - class Receiver - class ProcessingError < StandardError; end - class EmailUnparsableError < ProcessingError; end - class SentNotificationNotFoundError < ProcessingError; end - class EmptyEmailError < ProcessingError; end - class AutoGeneratedEmailError < ProcessingError; end - class UserNotFoundError < ProcessingError; end - class UserBlockedError < ProcessingError; end - class UserNotAuthorizedError < ProcessingError; end - class NoteableNotFoundError < ProcessingError; end - class InvalidNoteError < ProcessingError; end + class ProcessingError < StandardError; end + class EmailUnparsableError < ProcessingError; end + class SentNotificationNotFoundError < ProcessingError; end + class ProjectNotFound < ProcessingError; end + class EmptyEmailError < ProcessingError; end + class AutoGeneratedEmailError < ProcessingError; end + class UserNotFoundError < ProcessingError; end + class UserBlockedError < ProcessingError; end + class UserNotAuthorizedError < ProcessingError; end + class NoteableNotFoundError < ProcessingError; end + class InvalidNoteError < ProcessingError; end + class InvalidIssueError < ProcessingError; end + class UnknownIncomingEmail < ProcessingError; end + class Receiver def initialize(raw) @raw = raw end @@ -20,91 +26,38 @@ module Gitlab def execute raise EmptyEmailError if @raw.blank? - raise SentNotificationNotFoundError unless sent_notification - - raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/ - - author = sent_notification.recipient - - raise UserNotFoundError unless author - - raise UserBlockedError if author.blocked? - - project = sent_notification.project - - raise UserNotAuthorizedError unless project && author.can?(:create_note, project) - - raise NoteableNotFoundError unless sent_notification.noteable - - reply = ReplyParser.new(message).execute.strip - - raise EmptyEmailError if reply.blank? - - reply = add_attachments(reply) - - note = create_note(reply) + mail = build_mail + mail_key = extract_mail_key(mail) + handler = Handler.for(mail, mail_key) - unless note.persisted? - msg = "The comment could not be created for the following reasons:" - note.errors.full_messages.each do |error| - msg << "\n\n- #{error}" - end + raise UnknownIncomingEmail unless handler - raise InvalidNoteError, msg - end + handler.execute end - private - - def message - @message ||= Mail::Message.new(@raw) - rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e + def build_mail + Mail::Message.new(@raw) + rescue Encoding::UndefinedConversionError, + Encoding::InvalidByteSequenceError => e raise EmailUnparsableError, e end - def reply_key - key_from_to_header || key_from_additional_headers + def extract_mail_key(mail) + key_from_to_header(mail) || key_from_additional_headers(mail) end - def key_from_to_header - key = nil - message.to.each do |address| + def key_from_to_header(mail) + mail.to.find do |address| key = Gitlab::IncomingEmail.key_from_address(address) - break if key + break key if key end - - key end - def key_from_additional_headers - reply_key = nil - - Array(message.references).each do |message_id| - reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id) - break if reply_key + def key_from_additional_headers(mail) + Array(mail.references).find do |mail_id| + key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) + break key if key end - - reply_key - end - - def sent_notification - return nil unless reply_key - - SentNotification.for(reply_key) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) - - attachments.each do |link| - reply << "\n\n#{link[:markdown]}" - end - - reply - end - - def create_note(reply) - sent_notification.create_note(reply) end end end diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb deleted file mode 100644 index 93c6a5bb7f5..00000000000 --- a/lib/gitlab/force_push_check.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Gitlab - class ForcePushCheck - def self.force_push?(project, oldrev, newrev) - return false if project.empty_repo? - - # Created or deleted branch - if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - false - else - missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) - missed_refs.split("\n").size > 0 - end - end - end -end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 191bea86ac3..7584efe4fa8 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,24 @@ module Gitlab ref.gsub(/\Arefs\/(tags|heads)\//, '') end + def branch_name(ref) + ref = ref.to_s + if self.branch_ref?(ref) + self.ref_name(ref) + else + nil + end + end + + def tag_name(ref) + ref = ref.to_s + if self.tag_ref?(ref) + self.ref_name(ref) + else + nil + end + end + def tag_ref?(ref) ref.start_with?(TAG_REF_PREFIX) end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 7679c7e4bb8..1882eb8d050 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -1,66 +1,31 @@ +# Check a user's access to perform a git action. All public methods in this +# class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project, :protocol + attr_reader :actor, :project, :protocol, :user_access def initialize(actor, project, protocol) @actor = actor @project = project @protocol = protocol + @user_access = UserAccess.new(user, project: project) end - def user - return @user if defined?(@user) - - @user = - case actor - when User - actor - when DeployKey - nil - when Key - actor.user - end - end - - def deploy_key - actor if actor.is_a?(DeployKey) - end - - def can_push_to_branch?(ref) - return false unless user - - if project.protected_branch?(ref) && !project.developers_can_push_to_protected_branch?(ref) - user.can?(:push_code_to_protected_branches, project) - else - user.can?(:push_code, project) - end - end - - def can_read_project? - if user - user.can?(:read_project, project) - elsif deploy_key - deploy_key.projects.include?(project) - else - false - end - end - - def check(cmd, changes = nil) + def check(cmd, changes) return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? unless actor return build_status_object(false, "No user or key was provided.") end - if user && !user_allowed? + if user && !user_access.allowed? return build_status_object(false, "Your account has been blocked.") end - unless project && can_read_project? + unless project && (user_access.can_read_project? || deploy_key_can_read_project?) return build_status_object(false, 'The project you were looking for could not be found.') end @@ -95,7 +60,7 @@ module Gitlab end def user_download_access_check - unless user.can?(:download_code, project) + unless user_access.can_do_action?(:download_code) return build_status_object(false, "You are not allowed to download code from this project.") end @@ -111,10 +76,10 @@ module Gitlab return build_status_object(false, "A repository for this project does not exist yet.") end - changes = changes.lines if changes.kind_of?(String) + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied - changes.map(&:strip).reject(&:blank?).each do |change| + changes_list.each do |change| status = change_access_check(change) unless status.allowed? # If user does not have access to make at least one change - cancel all push @@ -125,46 +90,8 @@ module Gitlab build_status_object(true) end - def can_user_do_action?(action) - @permission_cache ||= {} - @permission_cache[action] ||= user.can?(action, project) - end - def change_access_check(change) - oldrev, newrev, ref = change.split(' ') - - action = - if project.protected_branch?(branch_name(ref)) - protected_branch_action(oldrev, newrev, branch_name(ref)) - elsif (tag_ref = tag_name(ref)) && protected_tag?(tag_ref) - # Prevent any changes to existing git tag unless user has permissions - :admin_project - else - :push_code - end - - unless can_user_do_action?(action) - status = - case action - when :force_push_code_to_protected_branches - build_status_object(false, "You are not allowed to force push code to a protected branch on this project.") - when :remove_protected_branches - build_status_object(false, "You are not allowed to deleted protected branches from this project.") - when :push_code_to_protected_branches - build_status_object(false, "You are not allowed to push code to protected branches on this project.") - when :admin_project - build_status_object(false, "You are not allowed to change existing tags on this project.") - else # :push_code - build_status_object(false, "You are not allowed to push code to this project.") - end - return status - end - - build_status_object(true) - end - - def forced_push?(oldrev, newrev) - Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) + Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec end def protocol_allowed? @@ -173,50 +100,41 @@ module Gitlab private - def protected_branch_action(oldrev, newrev, branch_name) - # we dont allow force push to protected branch - if forced_push?(oldrev, newrev) - :force_push_code_to_protected_branches - elsif Gitlab::Git.blank_ref?(newrev) - # and we dont allow remove of protected branch - :remove_protected_branches - elsif project.developers_can_push_to_protected_branch?(branch_name) - :push_code - else - :push_code_to_protected_branches - end + def matching_merge_request?(newrev, branch_name) + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end - def protected_tag?(tag_name) - project.repository.tag_exists?(tag_name) - end - - def user_allowed? - Gitlab::UserAccess.allowed?(user) - end - - def branch_name(ref) - ref = ref.to_s - if Gitlab::Git.branch_ref?(ref) - Gitlab::Git.ref_name(ref) - else - nil - end + def deploy_key + actor if actor.is_a?(DeployKey) end - def tag_name(ref) - ref = ref.to_s - if Gitlab::Git.tag_ref?(ref) - Gitlab::Git.ref_name(ref) + def deploy_key_can_read_project? + if deploy_key + return true if project.public? + deploy_key.projects.include?(project) else - nil + false end end protected + def user + return @user if defined?(@user) + + @user = + case actor + when User + actor + when DeployKey + nil + when Key + actor.user + end + end + def build_status_object(status, message = '') - GitAccessStatus.new(status, message) + Gitlab::GitAccessStatus.new(status, message) end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb index 5a806ff6e0d..09bb01be694 100644 --- a/lib/gitlab/git_access_status.rb +++ b/lib/gitlab/git_access_status.rb @@ -8,8 +8,8 @@ module Gitlab @message = message end - def to_json - { status: @status, message: @message }.to_json + def to_json(opts = nil) + { status: @status, message: @message }.to_json(opts) end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 8672cbc0ec4..f71d3575909 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,7 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess def change_access_check(change) - if user.can?(:create_wiki, project) + if user_access.can_do_action?(:create_wiki) build_status_object(true) else build_status_object(false, "You are not allowed to write to this project's wiki.") diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index a088e19d1e7..d32bdd86427 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -39,7 +39,6 @@ module Gitlab end def deserialize_changes(changes) - changes = Base64.decode64(changes) unless changes.include?(' ') changes = utf8_encode_changes(changes) changes.lines end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 7d2d545b84e..4750675ae9d 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -7,10 +7,6 @@ module Gitlab branch_exists? && commit_exists? end - def name - @name ||= exists? ? ref : "#{ref}-#{short_id}" - end - def valid? repo.present? end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 043f10d96a9..084e514492c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -78,10 +78,21 @@ module Gitlab def rate_limit api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? end def rate_limit_exceed? - rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS end def rate_limit_sleep_time diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb deleted file mode 100644 index db1fabaa18a..00000000000 --- a/lib/gitlab/github_import/hook_formatter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module GithubImport - class HookFormatter - EVENTS = %w[* create delete pull_request push].freeze - - attr_reader :raw - - delegate :id, :name, :active, to: :raw - - def initialize(raw) - @raw = raw - end - - def config - raw.config.attrs - end - - def valid? - (EVENTS & raw.events).any? && active - end - end - end -end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 3932fcb1eda..02ffb43d89b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,24 +3,30 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :project, :repo, :repo_url + attr_reader :client, :errors, :project, :repo, :repo_url def initialize(project) @project = project @repo = project.import_source @repo_url = project.import_url + @errors = [] if credentials @client = Client.new(credentials[:user]) - @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" end end def execute - import_labels && import_milestones && import_issues && - import_pull_requests && import_wiki + import_labels + import_milestones + import_issues + import_pull_requests + import_wiki + handle_errors + + true end private @@ -29,22 +35,37 @@ module Gitlab @credentials ||= project.import_data.credentials if project.import_data end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + def import_labels labels = client.labels(repo, per_page: 100) - labels.each { |raw| LabelFormatter.new(project, raw).create! } - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + labels.each do |raw| + begin + LabelFormatter.new(project, raw).create! + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end def import_milestones milestones = client.milestones(repo, state: :all, per_page: 100) - milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + milestones.each do |raw| + begin + MilestoneFormatter.new(project, raw).create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end def import_issues @@ -54,85 +75,55 @@ module Gitlab gh_issue = IssueFormatter.new(project, raw) if gh_issue.valid? - issue = gh_issue.create! - apply_labels(issue) - import_comments(issue) if gh_issue.has_comments? + begin + issue = gh_issue.create! + apply_labels(issue) + import_comments(issue) if gh_issue.has_comments? + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message end def import_pull_requests - disable_webhooks - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) - source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } - target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } - branches_removed = source_branches_removed | target_branches_removed - - restore_branches(branches_removed) - pull_requests.each do |pull_request| - merge_request = pull_request.create! - apply_labels(merge_request) - import_comments(merge_request) - import_comments_on_diff(merge_request) + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request = pull_request.create! + apply_labels(merge_request) + import_comments(merge_request) + import_comments_on_diff(merge_request) + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(pull_request) + end end - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message - ensure - clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks - end - - def disable_webhooks - update_webhooks(hooks, active: false) end - def clean_up_disabled_webhooks - update_webhooks(hooks, active: true) + def restore_source_branch(pull_request) + project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name) end - def update_webhooks(hooks, options) - hooks.each do |hook| - client.edit_hook(repo, hook.id, hook.name, hook.config, options) - end + def restore_target_branch(pull_request) + project.repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) end - def hooks - @hooks ||= - begin - client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - - # The GitHub Repository Webhooks API returns 404 for users - # without admin access to the repository when listing hooks. - # In this case we just want to return gracefully instead of - # spitting out an error and stop the import process. - rescue Octokit::NotFound - [] - end + def remove_branch(name) + project.repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :remove_branch, name: name } end - def restore_branches(branches) - branches.each do |name, sha| - client.create_ref(repo, "refs/heads/#{name}", sha) - end - - project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') - end - - def clean_up_restored_branches(branches) - branches.each do |name, _| - client.delete_ref(repo, "heads/#{name}") - project.repository.delete_branch(name) rescue Rugged::ReferenceError - end + def clean_up_restored_branches(pull_request) + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? project.repository.after_remove_branch end @@ -141,9 +132,10 @@ module Gitlab issue = client.issue(repo, issuable.iid) if issue.labels.count > 0 - label_ids = issue.labels.map do |raw| - Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) - end + label_ids = issue.labels + .map { |raw| LabelFormatter.new(project, raw).attributes } + .map { |attrs| Label.find_by(attrs).try(:id) } + .compact issuable.update_attribute(:label_ids, label_ids) end @@ -161,8 +153,12 @@ module Gitlab def create_comments(issuable, comments) comments.each do |raw| - comment = CommentFormatter.new(project, raw) - issuable.notes.create!(comment.attributes) + begin + comment = CommentFormatter.new(project, raw) + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end @@ -172,16 +168,12 @@ module Gitlab gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end - - true rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - raise Projects::ImportService::Error, e.message - else - true + errors << { type: :wiki, errors: e.message } end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index a4ea2210abd..04aa3664f64 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,8 +1,8 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter - delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true + delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true def attributes { @@ -33,17 +33,33 @@ module Gitlab end def valid? - source_branch.valid? && target_branch.valid? && !cross_project? + source_branch.valid? && target_branch.valid? end def source_branch @source_branch ||= BranchFormatter.new(project, raw_data.head) end + def source_branch_name + @source_branch_name ||= begin + source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}" + end + end + def target_branch @target_branch ||= BranchFormatter.new(project, raw_data.base) end + def target_branch_name + @target_branch_name ||= begin + target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}" + end + end + + def url + raw_data.url + end + private def assigned? @@ -68,10 +84,6 @@ module Gitlab raw_data.body || "" end - def cross_project? - source_branch_repo.id != target_branch_repo.id - end - def description formatter.author_line(author) + body end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 3f76ec97977..46d40f75be6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -15,31 +15,35 @@ module Gitlab end def execute - project_identifier = CGI.escape(project.import_source) - - # Issues && Comments - issues = client.issues(project_identifier) - - issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"]) - body += issue["description"] - - comments = client.issue_comments(project_identifier, issue["id"]) - - if comments.any? - body += @formatter.comments_header + ActiveRecord::Base.no_touching do + project_identifier = CGI.escape(project.import_source) + + # Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] + + comments = client.issue_comments(project_identifier, issue["id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + end + + project.issues.create!( + iid: issue["iid"], + description: body, + title: issue["title"], + state: issue["state"], + updated_at: issue["updated_at"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) end - - comments.each do |comment| - body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) - end - - project.issues.create!( - description: body, - title: issue["title"], - state: issue["state"], - author_id: gl_user_id(project, issue["author"]["id"]) - ) end true diff --git a/lib/gitlab/gitorious_import.rb b/lib/gitlab/gitorious_import.rb deleted file mode 100644 index 8d0132a744c..00000000000 --- a/lib/gitlab/gitorious_import.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module GitoriousImport - GITORIOUS_HOST = "https://gitorious.org" - end -end diff --git a/lib/gitlab/gitorious_import/client.rb b/lib/gitlab/gitorious_import/client.rb deleted file mode 100644 index 99fe5bdebfc..00000000000 --- a/lib/gitlab/gitorious_import/client.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module GitoriousImport - class Client - attr_reader :repo_list - - def initialize(repo_list) - @repo_list = repo_list - end - - def authorize_url(redirect_uri) - "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}" - end - - def repos - @repos ||= repo_names.map { |full_name| GitoriousImport::Repository.new(full_name) } - end - - def repo(id) - repos.find { |repo| repo.id == id } - end - - private - - def repo_names - repo_list.to_s.split(',').map(&:strip).reject(&:blank?) - end - end - end -end diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb deleted file mode 100644 index 8e22aa9286d..00000000000 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module GitoriousImport - class ProjectCreator - attr_reader :repo, :namespace, :current_user - - def initialize(repo, namespace, current_user) - @repo = repo - @namespace = namespace - @current_user = current_user - end - - def execute - ::Projects::CreateService.new( - current_user, - name: repo.name, - path: repo.path, - description: repo.description, - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel::PUBLIC, - import_type: "gitorious", - import_source: repo.full_name, - import_url: repo.import_url - ).execute - end - end - end -end diff --git a/lib/gitlab/gitorious_import/repository.rb b/lib/gitlab/gitorious_import/repository.rb deleted file mode 100644 index c88f1ae358d..00000000000 --- a/lib/gitlab/gitorious_import/repository.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module GitoriousImport - Repository = Struct.new(:full_name) do - def id - Digest::SHA1.hexdigest(full_name) - end - - def namespace - segments.first - end - - def path - segments.last - end - - def name - path.titleize - end - - def description - "" - end - - def import_url - "#{GITORIOUS_HOST}/#{full_name}.git" - end - - private - - def segments - full_name.split('/') - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index d4f12cb1df9..c5a11148d33 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -5,7 +5,7 @@ module Gitlab gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = help_shortcuts_path + gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 41296415e35..9360afedfcb 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap, repository: repository). + def self.highlight(blob_name, blob_content, repository: nil, plain: false) + new(blob_name, blob_content, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -13,30 +13,34 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - attr_reader :lexer - def initialize(blob_name, blob_content, repository: nil, nowrap: true) + def initialize(blob_name, blob_content, repository: nil) + @formatter = Rouge::Formatters::HTMLGitlab.new + @repository = repository @blob_name = blob_name @blob_content = blob_content - @repository = repository - @formatter = rouge_formatter(nowrap: nowrap) - - @lexer = custom_language || begin - Rouge::Lexer.guess(filename: blob_name, source: blob_content).new - rescue Rouge::Lexer::AmbiguousGuess => e - e.alternatives.sort_by(&:tag).first - end end def highlight(text, continue: true, plain: false) if plain - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + hl_lexer = Rouge::Lexers::PlainText + continue = false else - @formatter.format(@lexer.lex(text, continue: continue)).html_safe + hl_lexer = self.lexer end + + @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe rescue @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end + def lexer + @lexer ||= custom_language || begin + Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.sort_by(&:tag).first + end + end + private def custom_language @@ -46,16 +50,5 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end - - def rouge_formatter(options = {}) - options = options.reverse_merge( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) - - Rouge::Formatters::HTMLGitlab.new(options) - end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 588647e5adb..bb562bdcd2c 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.1' + VERSION = '0.1.3' + FILENAME_LIMIT = 50 def export_path(relative_path:) File.join(storage_path, relative_path) @@ -12,6 +13,10 @@ module Gitlab File.join(Settings.shared['path'], 'tmp/project_exports') end + def import_upload_path(filename:) + File.join(storage_path, 'uploads', filename) + end + def project_filename "project.json" end @@ -28,6 +33,12 @@ module Gitlab 'VERSION' end + def export_filename(project:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}" + + "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" + end + def version VERSION end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb new file mode 100644 index 00000000000..cfa595629f4 --- /dev/null +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class AvatarRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true unless avatar_export_file + + @project.avatar = File.open(avatar_export_file) + @project.save! + rescue => e + @shared.error(e) + false + end + + private + + def avatar_export_file + @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + end + + def avatar_export_path + File.join(@shared.export_path, 'avatar') + end + end + end +end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb new file mode 100644 index 00000000000..998c21e2586 --- /dev/null +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -0,0 +1,31 @@ +module Gitlab + module ImportExport + class AvatarSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless @project.avatar.exists? + + copy_files(avatar_path, avatar_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def avatar_export_path + File.join(@shared.export_path, 'avatar', @project.avatar_identifier) + end + + def avatar_path + @project.avatar.path + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2249904145c..e522a0fc8f6 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -17,6 +17,10 @@ module Gitlab execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) end + def git_restore_hooks + execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) + end + private def tar_with_options(archive:, dir:, options:) @@ -36,6 +40,19 @@ module Gitlab def git_bin_path Gitlab.config.git.bin_path end + + def copy_files(source, destination) + # if we are copying files, create the destination folder + destination_folder = File.file?(source) ? File.dirname(destination) : destination + + FileUtils.mkdir_p(destination_folder) + FileUtils.copy_entry(source, destination) + true + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 82d1e1805c5..eca6e5b6d51 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -3,6 +3,8 @@ module Gitlab class FileImporter include Gitlab::ImportExport::CommandLineUtil + MAX_RETRIES = 8 + def self.import(*args) new(*args).import end @@ -14,7 +16,10 @@ module Gitlab def import FileUtils.mkdir_p(@shared.export_path) - decompress_archive + + wait_for_archived_file do + decompress_archive + end rescue => e @shared.error(e) false @@ -22,6 +27,17 @@ module Gitlab private + # Exponentially sleep until I/O finishes copying the file + def wait_for_archived_file + MAX_RETRIES.times do |retry_number| + break if File.exist?(@archive_file) + + sleep(2**retry_number) + end + + yield + end + def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 05f4ad527ac..1da51043611 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -3,11 +3,12 @@ project_tree: - issues: - :events - notes: - - :author - - :events - - :labels - - milestones: - - :events + - :author + - :events + - label_links: + - :label + - milestone: + - :events - snippets: - notes: :author @@ -20,6 +21,10 @@ project_tree: - :events - :merge_request_diff - :events + - label_links: + - :label + - milestone: + - :events - pipelines: - notes: - :author @@ -31,6 +36,9 @@ project_tree: - :services - :hooks - :protected_branches + - :labels + - milestones: + - :events # Only include the following attributes for the models specified. included_attributes: @@ -53,7 +61,15 @@ included_attributes: excluded_attributes: snippets: - :expired_at + merge_request_diff: + - :st_diffs + issues: + - :milestone_id + merge_requests: + - :milestone_id methods: statuses: - - :type
\ No newline at end of file + - :type + merge_request_diff: + - :utf8_st_diffs
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 8f66f48cbfe..e9ee47fc090 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab end def execute - if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) @@ -35,6 +35,10 @@ module Gitlab project: @project) end + def avatar_restorer + Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: @shared, @@ -44,8 +48,7 @@ module Gitlab def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project), - wiki: true) + project: ProjectWiki.new(project_tree.restored_project)) end def uploads_restorer diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb new file mode 100644 index 00000000000..0cc10f40087 --- /dev/null +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -0,0 +1,107 @@ +module Gitlab + module ImportExport + # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json + # and its peculiar options. + class JsonHashBuilder + def self.build(model_objects, attributes_finder) + new(model_objects, attributes_finder).build + end + + def initialize(model_objects, attributes_finder) + @model_objects = model_objects + @attributes_finder = attributes_finder + end + + def build + process_model_objects(@model_objects) + end + + private + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def process_model_objects(model_object_hash) + json_config_hash = {} + current_key = model_object_hash.keys.first + + model_object_hash.values.flatten.each do |model_object| + @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } + handle_model_object(current_key, model_object, json_config_hash) + end + + json_config_hash + end + + # Creates or adds to an existing hash an individual model or list + # + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + # +json_config_hash+ the original hash containing the root model + def handle_model_object(current_key, model_object, json_config_hash) + model_or_sub_model = model_object.is_a?(Hash) ? process_model_objects(model_object) : model_object + + if json_config_hash[current_key] + add_model_value(current_key, model_or_sub_model, json_config_hash) + else + create_model_value(current_key, model_or_sub_model, json_config_hash) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + # +json_config_hash+ the original hash containing the root model + def create_model_value(current_key, value, json_config_hash) + json_config_hash[current_key] = parse_hash(value) || { include: value } + end + + # Calls attributes finder to parse the hash and add any attributes to it + # + # +value+ existing model to be included in the hash + # +parsed_hash+ the original hash + def parse_hash(value) + @attributes_finder.parse(value) do |hash| + { include: hash_or_merge(value, hash) } + end + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + # +json_config_hash+ the original hash containing the root model + def add_model_value(current_key, value, json_config_hash) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + + add_to_array(current_key, json_config_hash, value) + end + + # Adds new model configuration to an existing hash with key +current_key+ + # it creates a new array if it was previously a single value + # + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + # +json_config_hash+ the original hash containing the root model + def add_to_array(current_key, json_config_hash, value) + old_values = json_config_hash[current_key][:include] + + json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + # +value+ existing model to be included in the hash + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b459054c198..36c4cf6efa0 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -18,11 +18,14 @@ module Gitlab @map ||= begin @exported_members.inject(missing_keys_tracking_hash) do |hash, member| - existing_user = User.where(find_project_user_query(member)).first - old_user_id = member['user']['id'] - if existing_user && add_user_as_team_member(existing_user, member) - hash[old_user_id] = existing_user.id + if member['user'] + old_user_id = member['user']['id'] + existing_user = User.where(find_project_user_query(member)).first + hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user) + else + add_team_member(member) end + hash end end @@ -45,7 +48,7 @@ module Gitlab ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) end - def add_user_as_team_member(existing_user, member) + def add_team_member(member, existing_user = nil) member['user'] = existing_user ProjectMember.create(member_hash(member)).persisted? diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 0ac6ff01e3b..c7b3551b84c 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -12,7 +12,10 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') - create_relations + + ActiveRecord::Base.no_touching do + create_relations + end rescue => e @shared.error(e) false @@ -44,7 +47,7 @@ module Gitlab relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) - saved << restored_project.update_attribute(relation_key, relation_hash) + saved << restored_project.append_or_update_attribute(relation_key, relation_hash) end saved.all? end @@ -69,10 +72,19 @@ module Gitlab # Example: # +relation_key+ issues, loops through the list of *issues* and for each individual # issue, finds any subrelations such as notes, creates them and assign them back to the hash + # + # Recursively calls this method if the sub-relation is a hash containing more sub-relations def create_sub_relations(relation, tree_hash) relation_key = relation.keys.first.to_s - tree_hash[relation_key].each do |relation_item| + return if tree_hash[relation_key].blank? + + [tree_hash[relation_key]].flatten.each do |relation_item| relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author + + create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 15f5dd31035..5021a1a14ce 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -29,87 +29,12 @@ module Gitlab def build_hash(model_list) model_list.map do |model_objects| if model_objects.is_a?(Hash) - build_json_config_hash(model_objects) + Gitlab::ImportExport::JsonHashBuilder.build(model_objects, @attributes_finder) else @attributes_finder.find(model_objects) end end end - - # Called when the model is actually a hash containing other relations (more models) - # Returns the config in the right format for calling +to_json+ - # +model_object_hash+ - A model relationship such as: - # {:merge_requests=>[:merge_request_diff, :notes]} - def build_json_config_hash(model_object_hash) - @json_config_hash = {} - - model_object_hash.values.flatten.each do |model_object| - current_key = model_object_hash.keys.first - - @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } - - handle_model_object(current_key, model_object) - process_sub_model(current_key, model_object) if model_object.is_a?(Hash) - end - @json_config_hash - end - - # If the model is a hash, process the sub_models, which could also be hashes - # If there is a list, add to an existing array, otherwise use hash syntax - # +current_key+ main model that will be a key in the hash - # +model_object+ model or list of models to include in the hash - def process_sub_model(current_key, model_object) - sub_model_json = build_json_config_hash(model_object).dup - @json_config_hash.slice!(current_key) - - if @json_config_hash[current_key] && @json_config_hash[current_key][:include] - @json_config_hash[current_key][:include] << sub_model_json - else - @json_config_hash[current_key] = { include: sub_model_json } - end - end - - # Creates or adds to an existing hash an individual model or list - # +current_key+ main model that will be a key in the hash - # +model_object+ model or list of models to include in the hash - def handle_model_object(current_key, model_object) - if @json_config_hash[current_key] - add_model_value(current_key, model_object) - else - create_model_value(current_key, model_object) - end - end - - # Constructs a new hash that will hold the configuration for that particular object - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - def create_model_value(current_key, value) - parsed_hash = { include: value } - - @attributes_finder.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } - end - @json_config_hash[current_key] = parsed_hash - end - - # Adds new model configuration to an existing hash with key +current_key+ - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - def add_model_value(current_key, value) - @attributes_finder.parse(value) { |hash| value = { value => hash } } - old_values = @json_config_hash[current_key][:include] - @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten - end - - # Construct a new hash or merge with an existing one a model configuration - # This is to fulfil +to_json+ requirements. - # +value+ existing model to be included in the hash - # +hash+ hash containing configuration generated mainly from +@attributes_finder+ - def hash_or_merge(value, hash) - value.is_a?(Hash) ? value.merge(hash) : { value => hash } - end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 9824df3f274..b0726268ca6 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -13,6 +13,10 @@ module Gitlab BUILD_MODELS = %w[Ci::Build commit_status].freeze + IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + def self.create(*args) new(*args).create end @@ -22,23 +26,35 @@ module Gitlab @relation_hash = relation_hash.except('id', 'noteable_id') @members_mapper = members_mapper @user = user + @imported_object_retries = 0 end # Creates an object from an actual model with name "relation_sym" with params from # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create - set_note_author if @relation_name == :notes - update_user_references - update_project_references - reset_ci_tokens if @relation_name == 'Ci::Trigger' - @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] + setup_models generate_imported_object end private + def setup_models + if @relation_name == :notes + set_note_author + + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] + set_st_diffs if @relation_name == :merge_request_diff + end + def update_user_references USER_REFERENCES.each do |reference| if @relation_hash[reference] @@ -86,17 +102,19 @@ module Gitlab def update_project_references project_id = @relation_hash.delete('project_id') + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1 + end + # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['project_id'] = project_id @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] - @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + end - # If source and target are the same, populate them with the new project ID. - if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && - @relation_hash['target_project_id'] == @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = project_id - end + def same_source_and_target? + @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end def reset_ci_tokens @@ -111,10 +129,14 @@ module Gitlab end def imported_object - imported_object = relation_class.new(@relation_hash) - yield(imported_object) if block_given? - imported_object.importing = true if imported_object.respond_to?(:importing) - imported_object + yield(existing_or_new_object) if block_given? + existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + existing_or_new_object + rescue ActiveRecord::RecordNotUnique + # as the operation is not atomic, retry in the unlikely scenario an INSERT is + # performed on the same object between the SELECT and the INSERT + @imported_object_retries += 1 + retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES end def update_note_for_missing_author(author_name) @@ -125,6 +147,28 @@ module Gitlab def admin_user? @user.is_admin? end + + def parsed_relation_hash + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end + + def set_st_diffs + @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') + end + + def existing_or_new_object + # Only find existing records to avoid mapping tables such as milestones + # Otherwise always create the record, skipping the extra SELECT clause. + @existing_or_new_object ||= begin + if EXISTING_OBJECT_CHECK.include?(@relation_name) + existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) + existing_object.assign_attributes(parsed_relation_hash) + existing_object + else + relation_class.new(parsed_relation_hash) + end + end + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 546dae4d122..6d9379acf25 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,19 +3,18 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:, wiki: false) + def initialize(project:, shared:, path_to_bundle:) @project = project @path_to_bundle = path_to_bundle @shared = shared - @wiki = wiki end def restore - return wiki? unless File.exist?(@path_to_bundle) + return true unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) - git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks rescue => e @shared.error(e) false @@ -31,8 +30,14 @@ module Gitlab @project.repository.path_to_repo end + def repo_restore_hooks + return true if wiki? + + git_restore_hooks + end + def wiki? - @wiki + @project.class.name == 'ProjectWiki' end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index cce43fe994b..331e14021e6 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -11,7 +11,7 @@ module Gitlab end def save - return false if @project.empty_repo? + return true if @project.empty_repo? # it's ok to have no repo @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 6a60b65071f..6130c124dd1 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -7,7 +7,8 @@ module Gitlab new(*args).save end - def initialize(shared:) + def initialize(project:, shared:) + @project = project @shared = shared end @@ -36,7 +37,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project)) end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index d6f4fa57510..62a2553675c 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport class UploadsSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(project:, shared:) @project = project @shared = shared @@ -17,12 +19,6 @@ module Gitlab private - def copy_files(source, destination) - FileUtils.mkdir_p(destination) - FileUtils.copy_entry(source, destination) - true - end - def uploads_export_path File.join(@shared.export_path, 'uploads') end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index abfc694b879..de3fe6d822e 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -25,7 +25,7 @@ module Gitlab def verify_version!(version) if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") else true end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1eedae39f8a..6107420e4dd 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab def save @wiki = ProjectWiki.new(@project) return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 59a05411fe9..94261b7eeed 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -14,13 +14,12 @@ module Gitlab def options { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Gitorious.org' => 'gitorious', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', + 'GitHub' => 'github', + 'Bitbucket' => 'bitbucket', + 'GitLab.com' => 'gitlab', + 'Google Code' => 'google_code', + 'FogBugz' => 'fogbugz', + 'Repo by URL' => 'git', 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 8ce9d32abe0..d7be50bd437 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,7 +1,7 @@ module Gitlab module IncomingEmail class << self - FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze + FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze def enabled? config.enabled && config.address @@ -21,8 +21,8 @@ module Gitlab match[1] end - def key_from_fallback_reply_message_id(message_id) - match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX) + def key_from_fallback_message_id(mail_id) + match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX) return unless match match[1] diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index f2b649e50a2..2f326d00a2f 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -25,7 +25,7 @@ module Gitlab end end - def initialize(user, adapter=nil) + def initialize(user, adapter = nil) @adapter = adapter @user = user @provider = user.ldap_identity.provider diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index df65179bfea..9a5bcfb5c9b 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -13,7 +13,7 @@ module Gitlab Gitlab::LDAP::Config.new(provider) end - def initialize(provider, ldap=nil) + def initialize(provider, ldap = nil) @provider = provider @ldap = ldap || Net::LDAP.new(config.adapter_options) end diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb deleted file mode 100644 index 811363405a8..00000000000 --- a/lib/gitlab/lfs/response.rb +++ /dev/null @@ -1,327 +0,0 @@ -module Gitlab - module Lfs - class Response - def initialize(project, user, ci, request) - @origin_project = project - @project = storage_project(project) - @user = user - @ci = ci - @env = request.env - @request = request - end - - def render_download_object_response(oid) - render_response_to_download do - if check_download_sendfile_header? - render_lfs_sendfile(oid) - else - render_not_found - end - end - end - - def render_batch_operation_response - request_body = JSON.parse(@request.body.read) - case request_body["operation"] - when "download" - render_batch_download(request_body) - when "upload" - render_batch_upload(request_body) - else - render_not_found - end - end - - def render_storage_upload_authorize_response(oid, size) - render_response_to_push do - [ - 200, - { "Content-Type" => "application/json; charset=utf-8" }, - [JSON.dump({ - 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", - 'LfsOid' => oid, - 'LfsSize' => size - })] - ] - end - end - - def render_storage_upload_store_response(oid, size, tmp_file_name) - render_response_to_push do - render_lfs_upload_ok(oid, size, tmp_file_name) - end - end - - def render_unsupported_deprecated_api - [ - 501, - { "Content-Type" => "application/json; charset=utf-8" }, - [JSON.dump({ - 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - private - - def render_not_enabled - [ - 501, - { - "Content-Type" => "application/json; charset=utf-8", - }, - [JSON.dump({ - 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_unauthorized - [ - 401, - { - 'Content-Type' => 'text/plain' - }, - ['Unauthorized'] - ] - end - - def render_not_found - [ - 404, - { - "Content-Type" => "application/vnd.git-lfs+json" - }, - [JSON.dump({ - 'message' => 'Not found.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_forbidden - [ - 403, - { - "Content-Type" => "application/vnd.git-lfs+json" - }, - [JSON.dump({ - 'message' => 'Access forbidden. Check your access level.', - 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", - })] - ] - end - - def render_lfs_sendfile(oid) - return render_not_found unless oid.present? - - lfs_object = object_for_download(oid) - - if lfs_object && lfs_object.file.exists? - [ - 200, - { - # GitLab-workhorse will forward Content-Type header - "Content-Type" => "application/octet-stream", - "X-Sendfile" => lfs_object.file.path - }, - [] - ] - else - render_not_found - end - end - - def render_batch_upload(body) - return render_not_found if body.empty? || body['objects'].nil? - - render_response_to_push do - response = build_upload_batch_response(body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] - end - end - - def render_batch_download(body) - return render_not_found if body.empty? || body['objects'].nil? - - render_response_to_download do - response = build_download_batch_response(body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] - end - end - - def render_lfs_upload_ok(oid, size, tmp_file) - if store_file(oid, size, tmp_file) - [ - 200, - { - 'Content-Type' => 'text/plain', - 'Content-Length' => 0 - }, - [] - ] - else - [ - 422, - { 'Content-Type' => 'text/plain' }, - ["Unprocessable entity"] - ] - end - end - - def render_response_to_download - return render_not_enabled unless Gitlab.config.lfs.enabled - - unless @project.public? - return render_unauthorized unless @user || @ci - return render_forbidden unless user_can_fetch? - end - - yield - end - - def render_response_to_push - return render_not_enabled unless Gitlab.config.lfs.enabled - return render_unauthorized unless @user - return render_forbidden unless user_can_push? - - yield - end - - def check_download_sendfile_header? - @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" - end - - def user_can_fetch? - # Check user access against the project they used to initiate the pull - @ci || @user.can?(:download_code, @origin_project) - end - - def user_can_push? - # Check user access against the project they used to initiate the push - @user.can?(:push_code, @origin_project) - end - - def storage_project(project) - if project.forked? - storage_project(project.forked_from_project) - else - project - end - end - - def store_file(oid, size, tmp_file) - tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) - - object = LfsObject.find_or_create_by(oid: oid, size: size) - if object.file.exists? - success = true - else - success = move_tmp_file_to_storage(object, tmp_file_path) - end - - if success - success = link_to_project(object) - end - - success - ensure - # Ensure that the tmp file is removed - FileUtils.rm_f(tmp_file_path) - end - - def object_for_download(oid) - @project.lfs_objects.find_by(oid: oid) - end - - def move_tmp_file_to_storage(object, path) - File.open(path) do |f| - object.file = f - end - - object.file.store! - object.save - end - - def link_to_project(object) - if object && !object.projects.exists?(@project.id) - object.projects << @project - object.save - end - end - - def select_existing_objects(objects) - objects_oids = objects.map { |o| o['oid'] } - @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set - end - - def build_upload_batch_response(objects) - selected_objects = select_existing_objects(objects) - - upload_hypermedia_links(objects, selected_objects) - end - - def build_download_batch_response(objects) - selected_objects = select_existing_objects(objects) - - download_hypermedia_links(objects, selected_objects) - end - - def download_hypermedia_links(all_objects, existing_objects) - all_objects.each do |object| - if existing_objects.include?(object['oid']) - object['actions'] = { - 'download' => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - else - object['error'] = { - 'code' => 404, - 'message' => "Object does not exist on the server or you don't have permissions to access it", - } - end - end - - { 'objects' => all_objects } - end - - def upload_hypermedia_links(all_objects, existing_objects) - all_objects.each do |object| - # generate actions only for non-existing objects - next if existing_objects.include?(object['oid']) - - object['actions'] = { - 'upload' => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - end - - { 'objects' => all_objects } - end - end - end -end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb deleted file mode 100644 index 69bd5e62305..00000000000 --- a/lib/gitlab/lfs/router.rb +++ /dev/null @@ -1,100 +0,0 @@ -module Gitlab - module Lfs - class Router - attr_reader :project, :user, :ci, :request - - def initialize(project, user, ci, request) - @project = project - @user = user - @ci = ci - @env = request.env - @request = request - end - - def try_call - return unless @request && @request.path.present? - - case @request.request_method - when 'GET' - get_response - when 'POST' - post_response - when 'PUT' - put_response - else - nil - end - end - - private - - def get_response - path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) - return nil unless path_match - - oid = path_match[2] - return nil unless oid - - case path_match[1] - when "info/lfs" - lfs.render_unsupported_deprecated_api - when "gitlab-lfs" - lfs.render_download_object_response(oid) - else - nil - end - end - - def post_response - post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) - return nil unless post_path - - # Check for Batch API - if post_path[0].ends_with?("/info/lfs/objects/batch") - lfs.render_batch_operation_response - elsif post_path[0].ends_with?("/info/lfs/objects") - lfs.render_unsupported_deprecated_api - else - nil - end - end - - def put_response - object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) - return nil if object_match.nil? - - oid = object_match[1] - size = object_match[2].try(:to_i) - return nil if oid.nil? || size.nil? - - # GitLab-workhorse requests - # 1. Try to authorize the request - # 2. send a request with a header containing the name of the temporary file - if object_match[3] && object_match[3] == '/authorize' - lfs.render_storage_upload_authorize_response(oid, size) - else - tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) - return nil unless tmp_file_name - - lfs.render_storage_upload_store_response(oid, size, tmp_file_name) - end - end - - def lfs - return unless @project - - Gitlab::Lfs::Response.new(@project, @user, @ci, @request) - end - - def sanitize_tmp_filename(name) - if name.present? - name.gsub!(/^.*(\\|\/)/, '') - name = name.match(/[0-9a-f]{73}/) - name[0] if name - else - nil - end - end - end - end -end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb new file mode 100644 index 00000000000..12999a90a29 --- /dev/null +++ b/lib/gitlab/mail_room.rb @@ -0,0 +1,47 @@ +require 'yaml' +require 'json' +require_relative 'redis' unless defined?(Gitlab::Redis) + +module Gitlab + module MailRoom + class << self + def enabled? + config[:enabled] && config[:address] + end + + def config + @config ||= fetch_config + end + + def reset_config! + @config = nil + end + + private + + def fetch_config + return {} unless File.exist?(config_file) + + rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys + + config = all_config[:incoming_email] || {} + config[:enabled] = false if config[:enabled].nil? + config[:port] = 143 if config[:port].nil? + config[:ssl] = false if config[:ssl].nil? + config[:start_tls] = false if config[:start_tls].nil? + config[:mailbox] = 'inbox' if config[:mailbox].nil? + + if config[:enabled] && config[:address] + config[:redis_url] = Gitlab::Redis.new(rails_env).url + end + + config + end + + def config_file + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + end + end + end +end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 49f702f91f6..3d1ba33ec68 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -124,6 +124,20 @@ module Gitlab trans.action = action if trans end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + trans = current_transaction + + trans.add_event(*args) if trans + end + + # Returns the prefix to use for the name of a series. + def self.series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -136,8 +150,7 @@ module Gitlab end end - private - + # Allow access from other metrics related middlewares def self.current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index dcec7543c13..4b7a791e497 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -9,14 +9,17 @@ module Gitlab # # Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login) module Instrumentation - SERIES = 'method_calls' - PROXY_IVAR = :@__gitlab_instrumentation_proxy def self.configure yield self end + # Returns the name of the series to use for storing method calls. + def self.series + @series ||= "#{Metrics.series_prefix}method_calls" + end + # Instruments a class method. # # mod - The module to instrument as a Module/Class. @@ -141,15 +144,15 @@ module Gitlab # generated method _only_ accepts regular arguments if the underlying # method also accepts them. if method.arity == 0 - args_signature = '&block' + args_signature = '' else - args_signature = '*args, &block' + args_signature = '*args' end proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) if trans = Gitlab::Metrics::Instrumentation.transaction - trans.measure_method(#{label.inspect}) { super } + trans.method_call_for(#{label.to_sym.inspect}).measure { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index c048fe20ba7..d3465e5ec19 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -11,8 +11,8 @@ module Gitlab def initialize(name, series) @name = name @series = series - @real_time = 0.0 - @cpu_time = 0.0 + @real_time = 0 + @cpu_time = 0 @call_count = 0 end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f23d67e1e38..bd0afe53c51 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,15 +4,20 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags + attr_reader :series, :values, :tags, :type # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}) + def initialize(series, values, tags = {}, type = :metric) @values = values @series = series @tags = tags + @type = type + end + + def event? + type == :event end # Returns a Hash in a format that can be directly written to InfluxDB. diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e61670f491c..01c96a6fe96 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,6 +4,17 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' + CONTENT_TYPES = { + 'text/html' => :html, + 'text/plain' => :txt, + 'application/json' => :json, + 'text/js' => :js, + 'application/atom+xml' => :atom, + 'image/png' => :png, + 'image/jpeg' => :jpeg, + 'image/gif' => :gif, + 'image/svg+xml' => :svg + } def initialize(app) @app = app @@ -17,6 +28,10 @@ module Gitlab begin retval = trans.run { @app.call(env) } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:rails_exception) + + raise error # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure @@ -42,8 +57,15 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - trans.action = "#{controller.class.name}##{controller.action_name}" + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + suffix = CONTENT_TYPES[controller.content_type] + + if suffix && suffix != :html + action += ".#{suffix}" + end + + trans.action = action end def tag_endpoint(trans, env) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index a1240fd33ee..f9dd8e41912 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,10 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:sidekiq_exception) + + raise error ensure trans.finish end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 82c18bb108b..287b7a83547 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -35,12 +35,12 @@ module Gitlab if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time Process. - clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) end else def self.cpu_time Process. - clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) end end @@ -48,14 +48,14 @@ module Gitlab # # Returns the time as a Float. def self.real_time(precision = :millisecond) - Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + Process.clock_gettime(Process::CLOCK_REALTIME, precision) end # Returns the current monotonic clock time in a given precision. # # Returns the time as a Float. def self.monotonic_time(precision = :millisecond) - Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision) end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index bded245da43..7bc16181be6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,10 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values, :methods + # The series to store events (e.g. Git pushes) in. + EVENT_SERIES = 'events' + + attr_reader :tags, :values, :method, :metrics attr_accessor :action @@ -52,23 +55,30 @@ module Gitlab end def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) end - # Measures the time it takes to execute a method. + # Tracks a business level event # - # Multiple calls to the same method add up to the total runtime of the - # method. + # Business level events including events such as Git pushes, Emails being + # sent, etc. # - # name - The full name of the method to measure (e.g. `User#sign_in`). - def measure_method(name, &block) - unless @methods[name] - series = "#{series_prefix}#{Instrumentation::SERIES}" + # event_name - The name of the event (e.g. "git_push"). + # tags - A set of tags to attach to the event. + def add_event(event_name, tags = {}) + @metrics << Metric.new(EVENT_SERIES, + { count: 1 }, + { event: event_name }.merge(tags), + :event) + end - @methods[name] = MethodCall.new(name, series) + # Returns a MethodCall object for the given name. + def method_call_for(name) + unless method = @methods[name] + @methods[name] = method = MethodCall.new(name, Instrumentation.series) end - @methods[name].measure(&block) + method end def increment(name, value) @@ -108,21 +118,13 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - hash[:tags][:action] ||= @action if @action + hash[:tags][:action] ||= @action if @action && !metric.event? hash end Metrics.submit_metrics(submit_hashes) end - - def sidekiq? - Sidekiq.server? - end - - def series_prefix - sidekiq? ? 'sidekiq_' : 'rails_' - end end end end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 56608b1b276..5d2d7d0026c 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -11,7 +11,7 @@ module Gitlab def call(env) trans = Gitlab::Metrics.current_transaction - proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index 43e07e09160..ca23ccef25b 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,7 +5,7 @@ module Gitlab module Popen extend self - def popen(cmd, path=nil) + def popen(cmd, path = nil) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 40766f35f77..9376b54f43b 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -1,50 +1,94 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' + module Gitlab class Redis CACHE_NAMESPACE = 'cache:gitlab' SESSION_NAMESPACE = 'session:gitlab' SIDEKIQ_NAMESPACE = 'resque:gitlab' - - attr_reader :url + MAILROOM_NAMESPACE = 'mail_room:gitlab' + DEFAULT_REDIS_URL = 'redis://localhost:6379' # To be thread-safe we must be careful when writing the class instance # variables @url and @pool. Because @pool depends on @url we need two # mutexes to prevent deadlock. - URL_MUTEX = Mutex.new + PARAMS_MUTEX = Mutex.new POOL_MUTEX = Mutex.new - private_constant :URL_MUTEX, :POOL_MUTEX + private_constant :PARAMS_MUTEX, :POOL_MUTEX - def self.url - @url || URL_MUTEX.synchronize { @url = new.url } - end + class << self + def params + @params || PARAMS_MUTEX.synchronize { @params = new.params } + end + + # @deprecated Use .params instead to get sentinel support + def url + new.url + end - def self.with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(url: url) } + def with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(params) } + end end + @pool.with { |redis| yield redis } + end + + def reset_params! + @params = nil end - @pool.with { |redis| yield redis } end - def self.redis_store_options - url = new.url - redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) end - redis_config_hash end - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) + def raw_config_hash + config_data = fetch_config - @url = "redis://localhost:6379" - if File.exist?(config_file) - @url = YAML.load_file(config_file)[rails_env] + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: DEFAULT_REDIS_URL } end end + + def fetch_config + file = config_file + File.exist?(file) ? YAML.load_file(file)[@rails_env] : false + end + + def config_file + File.expand_path('../../../config/resque.yml', __FILE__) + end end end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb new file mode 100644 index 00000000000..8130e55351e --- /dev/null +++ b/lib/gitlab/request_profiler.rb @@ -0,0 +1,19 @@ +require 'fileutils' + +module Gitlab + module RequestProfiler + PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" + + def profile_token + Rails.cache.fetch('profile-token') do + Devise.friendly_token + end + end + module_function :profile_token + + def remove_all_profiles + FileUtils.rm_rf(PROFILES_DIR) + end + module_function :remove_all_profiles + end +end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb new file mode 100644 index 00000000000..786e1d49f5e --- /dev/null +++ b/lib/gitlab/request_profiler/middleware.rb @@ -0,0 +1,54 @@ +require 'ruby-prof' +require_dependency 'gitlab/request_profiler' + +module Gitlab + module RequestProfiler + class Middleware + def initialize(app) + @app = app + end + + def call(env) + if profile?(env) + call_with_profiling(env) + else + @app.call(env) + end + end + + def profile?(env) + header_token = env['HTTP_X_PROFILE_TOKEN'] + return unless header_token.present? + + profile_token = RequestProfiler.profile_token + return unless profile_token.present? + + header_token == profile_token + end + + def call_with_profiling(env) + ret = nil + result = RubyProf::Profile.profile do + ret = catch(:warden) do + @app.call(env) + end + end + + printer = RubyProf::CallStackPrinter.new(result) + file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html" + file_path = "#{PROFILES_DIR}/#{file_name}" + + FileUtils.mkdir_p(PROFILES_DIR) + File.open(file_path, 'wb') do |file| + printer.print(file) + end + + if ret.is_a?(Array) + ret + else + throw(:warden, ret) + end + end + end + end +end diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb new file mode 100644 index 00000000000..f89d56903ef --- /dev/null +++ b/lib/gitlab/request_profiler/profile.rb @@ -0,0 +1,43 @@ +module Gitlab + module RequestProfiler + class Profile + attr_reader :name, :time, :request_path + + alias_method :to_param, :name + + def self.all + Dir["#{PROFILES_DIR}/*.html"].map do |path| + new(File.basename(path)) + end + end + + def self.find(name) + name_dup = name.dup + name_dup << '.html' unless name.end_with?('.html') + + file_path = "#{PROFILES_DIR}/#{name_dup}" + return unless File.exist?(file_path) + + new(name_dup) + end + + def initialize(name) + @name = name + + set_attributes + end + + def content + File.read("#{PROFILES_DIR}/#{name}") + end + + private + + def set_attributes + _, path, timestamp = name.split(/(.*)_(\d+)\.html$/) + @request_path = path.tr('|', '/') + @time = Time.at(timestamp.to_i).utc + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb new file mode 100644 index 00000000000..b1fa0e3cb4e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqMiddleware + class RequestStoreMiddleware + def call(worker, job, queue) + RequestStore.begin! + yield + ensure + RequestStore.end! + RequestStore.clear! + end + end + end +end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb new file mode 100644 index 00000000000..60d35be2599 --- /dev/null +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -0,0 +1,57 @@ +module Gitlab + module SlashCommands + class CommandDefinition + attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + + def initialize(name, attributes = {}) + @name = name + + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @params = attributes[:params] || [] + @condition_block = attributes[:condition_block] + @action_block = attributes[:action_block] + end + + def all_names + [name, *aliases] + end + + def noop? + action_block.nil? + end + + def available?(opts) + return true unless condition_block + + context = OpenStruct.new(opts) + context.instance_exec(&condition_block) + end + + def execute(context, opts, arg) + return if noop? || !available?(opts) + + if arg.present? + context.instance_exec(arg, &action_block) + elsif action_block.arity == 0 + context.instance_exec(&action_block) + end + end + + def to_h(opts) + desc = description + if desc.respond_to?(:call) + context = OpenStruct.new(opts) + desc = context.instance_exec(&desc) rescue '' + end + + { + name: name, + aliases: aliases, + description: desc, + params: params + } + end + end + end +end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb new file mode 100644 index 00000000000..50b0937d267 --- /dev/null +++ b/lib/gitlab/slash_commands/dsl.rb @@ -0,0 +1,98 @@ +module Gitlab + module SlashCommands + module Dsl + extend ActiveSupport::Concern + + included do + cattr_accessor :command_definitions, instance_accessor: false do + [] + end + + cattr_accessor :command_definitions_by_name, instance_accessor: false do + {} + end + end + + class_methods do + # Allows to give a description to the next slash command. + # This description is shown in the autocomplete menu. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # desc do + # "This is a dynamic description for #{noteable.to_ability_name}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def desc(text = '', &block) + @description = block_given? ? block : text + end + + # Allows to define params for the next slash command. + # These params are shown in the autocomplete menu. + # + # Example: + # + # params "~label ~label2" + # command :command_key do |arguments| + # # Awesome code block + # end + def params(*params) + @params = params + end + + # Allows to define conditions that must be met in order for the command + # to be returned by `.command_names` & `.command_definitions`. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # condition do + # project.public? + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def condition(&block) + @condition_block = block + end + + # Registers a new command which is recognizeable from body of email or + # comment. + # It accepts aliases and takes a block. + # + # Example: + # + # command :my_command, :alias_for_my_command do |arguments| + # # Awesome code block + # end + def command(*command_names, &block) + name, *aliases = command_names + + definition = CommandDefinition.new( + name, + aliases: aliases, + description: @description, + params: @params, + condition_block: @condition_block, + action_block: block + ) + + self.command_definitions << definition + + definition.all_names.each do |name| + self.command_definitions_by_name[name] = definition + end + + @description = nil + @params = nil + @condition_block = nil + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb new file mode 100644 index 00000000000..a672e5e4855 --- /dev/null +++ b/lib/gitlab/slash_commands/extractor.rb @@ -0,0 +1,122 @@ +module Gitlab + module SlashCommands + # This class takes an array of commands that should be extracted from a + # given text. + # + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # ``` + class Extractor + attr_reader :command_definitions + + def initialize(command_definitions) + @command_definitions = command_definitions + end + + # Extracts commands from content and return an array of commands. + # The array looks like the following: + # [ + # ['command1'], + # ['command3', 'arg1 arg2'], + # ] + # The command and the arguments are stripped. + # The original command text is removed from the given `content`. + # + # Usage: + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) + # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] + # msg #=> "hello\nworld" + # ``` + def extract_commands(content, opts = {}) + return [content, []] unless content + + content = content.dup + + commands = [] + + content.delete!("\r") + content.gsub!(commands_regex(opts)) do + if $~[:cmd] + commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + '' + else + $~[0] + end + end + + [content.strip, commands] + end + + private + + # Builds a regular expression to match known commands. + # First match group captures the command name and + # second match group captures its arguments. + # + # It looks something like: + # + # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ + def commands_regex(opts) + names = command_names(opts).map(&:to_s) + + @commands_regex ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `/cmd arg` which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `/cmd arg` which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?<html> + # Quote block: + # >>> + # Anything, including `/cmd arg` which are ignored by this filter + # >>> + + ^>>> + .+? + \n>>>$ + ) + | + (?: + # Command not in a blockquote, blockcode, or HTML tag: + # /close + + ^\/ + (?<cmd>#{Regexp.union(names)}) + (?: + [ ] + (?<arg>[^\/\n]*) + )? + (?:\n|$) + ) + }mx + end + + def command_names(opts) + command_definitions.flat_map do |command| + next if command.noop? + + command.all_names + end.compact + end + end + end +end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 760ff3e614a..7ebec8e2cff 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,8 +1,9 @@ module Gitlab module Template class BaseTemplate - def initialize(path) + def initialize(path, project = nil) @path = path + @finder = self.class.finder(project) end def name @@ -10,23 +11,32 @@ module Gitlab end def content - File.read(@path) + @finder.read(@path) + end + + def to_json + { name: name, content: content } end class << self - def all - self.categories.keys.flat_map { |cat| by_category(cat) } + def all(project = nil) + if categories.any? + categories.keys.flat_map { |cat| by_category(cat, project) } + else + by_category("", project) + end end - def find(key) - file_name = "#{key}#{self.extension}" - - directory = select_directory(file_name) - directory ? new(File.join(category_directory(directory), file_name)) : nil + def find(key, project = nil) + path = self.finder(project).find(key) + path.present? ? new(path, project) : nil end + # Set categories as sub directories + # Example: { "category_name_1" => "directory_path_1", "category_name_2" => "directory_name_2" } + # Default is no category with all files in base dir of each class def categories - raise NotImplementedError + {} end def extension @@ -37,29 +47,40 @@ module Gitlab raise NotImplementedError end - def by_category(category) - templates_for_directory(category_directory(category)) + # Defines which strategy will be used to get templates files + # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject + # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects + def finder(project = nil) + raise NotImplementedError end - def category_directory(category) - File.join(base_dir, categories[category]) + def by_category(category, project = nil) + directory = category_directory(category) + files = finder(project).list_files_for(directory) + + files.map { |f| new(f, project) } end - private + def category_directory(category) + return base_dir unless category.present? - def select_directory(file_name) - categories.keys.find do |category| - File.exist?(File.join(category_directory(category), file_name)) - end + File.join(base_dir, categories[category]) end - def templates_for_directory(dir) - dir << '/' unless dir.end_with?('/') - Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) } - end + # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] } + # If no category is present returns [{ name: template_name }, { name: template2_name}] + def dropdown_names(project = nil) + return [] if project && !project.repository.exists? - def filter_regex - @filter_reges ||= /#{Regexp.escape(extension)}\z/ + if categories.any? + categories.keys.map do |category| + files = self.by_category(category, project) + [category, files.map { |t| { name: t.name } }] + end.to_h + else + files = self.all(project) + files.map { |t| { name: t.name } } + end end end end diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb new file mode 100644 index 00000000000..473b05257c6 --- /dev/null +++ b/lib/gitlab/template/finders/base_template_finder.rb @@ -0,0 +1,35 @@ +module Gitlab + module Template + module Finders + class BaseTemplateFinder + def initialize(base_dir) + @base_dir = base_dir + end + + def list_files_for + raise NotImplementedError + end + + def read + raise NotImplementedError + end + + def find + raise NotImplementedError + end + + def category_directory(category) + return @base_dir unless category.present? + + @base_dir + @categories[category] + end + + class << self + def filter_regex(extension) + /#{Regexp.escape(extension)}\z/ + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb new file mode 100644 index 00000000000..831da45191f --- /dev/null +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -0,0 +1,38 @@ +# Searches and reads file present on Gitlab installation directory +module Gitlab + module Template + module Finders + class GlobalTemplateFinder < BaseTemplateFinder + def initialize(base_dir, extension, categories = {}) + @categories = categories + @extension = extension + super(base_dir) + end + + def read(path) + File.read(path) + end + + def find(key) + file_name = "#{key}#{@extension}" + + directory = select_directory(file_name) + directory ? File.join(category_directory(directory), file_name) : nil + end + + def list_files_for(dir) + dir << '/' unless dir.end_with?('/') + Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + @categories.keys.find do |category| + File.exist?(File.join(category_directory(category), file_name)) + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb new file mode 100644 index 00000000000..22c39436cb2 --- /dev/null +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -0,0 +1,59 @@ +# Searches and reads files present on each Gitlab project repository +module Gitlab + module Template + module Finders + class RepoTemplateFinder < BaseTemplateFinder + # Raised when file is not found + class FileNotFoundError < StandardError; end + + def initialize(project, base_dir, extension, categories = {}) + @categories = categories + @extension = extension + @repository = project.repository + @commit = @repository.head_commit if @repository.exists? + + super(base_dir) + end + + def read(path) + blob = @repository.blob_at(@commit.id, path) if @commit + raise FileNotFoundError if blob.nil? + blob.data + end + + def find(key) + file_name = "#{key}#{@extension}" + directory = select_directory(file_name) + raise FileNotFoundError if directory.nil? + + category_directory(directory) + file_name + end + + def list_files_for(dir) + return [] unless @commit + + dir << '/' unless dir.end_with?('/') + + entries = @repository.tree(:head, dir).entries + + names = entries.map(&:name) + names.select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + return [] unless @commit + + # Insert root as directory + directories = ["", @categories.keys] + + directories.find do |category| + path = category_directory(category) + file_name + @repository.blob_at(@commit.id, path) + end + end + end + end + end +end diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore_template.rb index 964fbfd4de3..8d2a9d2305c 100644 --- a/lib/gitlab/template/gitignore.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class Gitignore < BaseTemplate + class GitignoreTemplate < BaseTemplate class << self def extension '.gitignore' @@ -16,6 +16,10 @@ module Gitlab def base_dir Rails.root.join('vendor/gitignore') end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 7f480fe33c0..8d1a1ed54c9 100644 --- a/lib/gitlab/template/gitlab_ci_yml.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class GitlabCiYml < BaseTemplate + class GitlabCiYmlTemplate < BaseTemplate def content explanation = "# This file is a template, and might need editing before it works on your project." [explanation, super].join("\n") @@ -21,6 +21,10 @@ module Gitlab def base_dir Rails.root.join('vendor/gitlab-ci-yml') end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end end end end diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb new file mode 100644 index 00000000000..c6fa8d3eafc --- /dev/null +++ b/lib/gitlab/template/issue_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class IssueTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/issue_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb new file mode 100644 index 00000000000..f826c02f3b5 --- /dev/null +++ b/lib/gitlab/template/merge_request_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class MergeRequestTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/merge_request_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 83f91de810c..d4020af76f9 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -2,6 +2,8 @@ module Gitlab # Module containing GitLab's application theme definitions and helper methods # for accessing them. module Themes + extend self + # Theme ID used when no `default_theme` configuration setting is provided. APPLICATION_DEFAULT = 2 @@ -22,7 +24,7 @@ module Gitlab # classes that might be applied to the `body` element # # Returns a String - def self.body_classes + def body_classes THEMES.collect(&:css_class).uniq.join(' ') end @@ -33,26 +35,26 @@ module Gitlab # id - Integer ID # # Returns a Theme - def self.by_id(id) + def by_id(id) THEMES.detect { |t| t.id == id } || default end # Returns the number of defined Themes - def self.count + def count THEMES.size end # Get the default Theme # # Returns a Theme - def self.default + def default by_id(default_id) end # Iterate through each Theme # # Yields the Theme object - def self.each(&block) + def each(&block) THEMES.each(&block) end @@ -61,7 +63,7 @@ module Gitlab # user - User record # # Returns a Theme - def self.for_user(user) + def for_user(user) if user by_id(user.theme_id) else @@ -71,7 +73,7 @@ module Gitlab private - def self.default_id + def default_id id = Gitlab.config.gitlab.default_theme.to_i # Prevent an invalid configuration setting from causing an infinite loop diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index fe65c246101..99d0c28e749 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -22,6 +22,8 @@ module Gitlab note_url when WikiPage wiki_page_url + when ProjectSnippet + project_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index d1b42c1f9b9..9858d2e7d83 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,7 +1,23 @@ module Gitlab - module UserAccess - def self.allowed?(user) - return false if user.blocked? + class UserAccess + attr_reader :user, :project + + def initialize(user, project: nil) + @user = user + @project = project + end + + def can_do_action?(action) + @permission_cache ||= {} + @permission_cache[action] ||= user.can?(action, project) + end + + def cannot_do_action?(action) + !can_do_action?(action) + end + + def allowed? + return false if user.blank? || user.blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -9,5 +25,35 @@ module Gitlab true end + + def can_push_to_branch?(ref) + return false unless user + + if project.protected_branch?(ref) + return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) + + access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten + access_levels.any? { |access_level| access_level.check_access(user) } + else + user.can?(:push_code, project) + end + end + + def can_merge_to_branch?(ref) + return false unless user + + if project.protected_branch?(ref) + access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten + access_levels.any? { |access_level| access_level.check_access(user) } + else + user.can?(:push_code, project) + end + end + + def can_read_project? + return false unless user + + user.can?(:read_project, project) + end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index d13fe0ef8a9..e59ead5d76c 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -7,7 +7,7 @@ module Gitlab # @param cmd [Array<String>] # @return [Boolean] def system_silent(cmd) - Popen::popen(cmd).last.zero? + Popen.popen(cmd).last.zero? end def force_utf8(str) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index bc0193a6c32..c6826a09bd2 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -4,6 +4,7 @@ require 'json' module Gitlab class Workhorse SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' + VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' class << self def git_http_ok(repository, user) @@ -63,6 +64,23 @@ module Gitlab ] end + def send_artifacts_entry(build, entry) + params = { + 'Archive' => build.artifacts_file.path, + 'Entry' => Base64.encode64(entry.path) + } + + [ + SEND_DATA_HEADER, + "artifacts-entry:#{encode(params)}" + ] + end + + def version + path = Rails.root.join(VERSION_FILE) + path.readable? ? path.read.chomp : 'unknown' + end + protected def encode(hash) diff --git a/lib/repository_cache.rb b/lib/repository_cache.rb index 8ddc3511293..068a95790c0 100644 --- a/lib/repository_cache.rb +++ b/lib/repository_cache.rb @@ -1,14 +1,15 @@ # Interface to the Redis-backed cache store used by the Repository model class RepositoryCache - attr_reader :namespace, :backend + attr_reader :namespace, :backend, :project_id - def initialize(namespace, backend = Rails.cache) + def initialize(namespace, project_id, backend = Rails.cache) @namespace = namespace @backend = backend + @project_id = project_id end def cache_key(type) - "#{type}:#{namespace}" + "#{type}:#{namespace}:#{project_id}" end def expire(key) diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 3358ed6773e..4edfd015074 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -1,171 +1,27 @@ -require 'cgi' - module Rouge module Formatters - class HTMLGitlab < Rouge::Formatter + class HTMLGitlab < Rouge::Formatters::HTML tag 'html_gitlab' # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # - # [+nowrap+] If set to True, don't wrap the output at all, not - # even inside a <tt><pre></tt> tag (default: false). - # [+cssclass+] CSS class for the wrapping <tt><div></tt> tag - # (default: 'highlight'). - # [+linenos+] If set to 'table', output line numbers as a table - # with two cells, one containing the line numbers, - # the other the whole code. This is copy paste friendly, - # but may cause alignment problems with some browsers - # or fonts. If set to 'inline', the line numbers will - # be integrated in the <tt><pre></tt> tag that contains - # the code (default: nil). # [+linenostart+] The line number for the first line (default: 1). - # [+lineanchors+] If set to true the formatter will wrap each output - # line in an anchor tag with a name of L-linenumber. - # This allows easy linking to certain lines - # (default: false). - # [+lineanchorsid+] If lineanchors is true the name of the anchors can - # be changed with lineanchorsid to e.g. foo-linenumber - # (default: 'L'). - # [+anchorlinenos+] If set to true, will wrap line numbers in <tt><a></tt> - # tags. Used in combination with linenos and lineanchors - # (default: false). - # [+inline_theme+] Inline CSS styles for the <pre> tag (default: false). - def initialize( - nowrap: false, - cssclass: 'highlight', - linenos: nil, - linenostart: 1, - lineanchors: false, - lineanchorsid: 'L', - anchorlinenos: false, - inline_theme: nil - ) - @nowrap = nowrap - @cssclass = cssclass - @linenos = linenos + def initialize(linenostart: 1) @linenostart = linenostart - @lineanchors = lineanchors - @lineanchorsid = lineanchorsid - @anchorlinenos = anchorlinenos - @inline_theme = Theme.find(inline_theme).new if inline_theme.is_a?(String) - end - - def render(tokens) - case @linenos - when 'table' - render_tableized(tokens) - when 'inline' - render_untableized(tokens) - else - render_untableized(tokens) - end - end - - alias_method :format, :render - - private - - def render_untableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap - html << wrap_lines(data[:code]) - html << "</code></pre>\n" unless @nowrap - html + @line_number = linenostart end - def render_tableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<div class=\"#{@cssclass}\">" unless @nowrap - html << '<table><tbody>' - html << "<td class=\"linenos\"><pre>" - html << wrap_linenos(data[:numbers]) - html << '</pre></td>' - html << "<td class=\"lines\"><pre><code>" - html << wrap_lines(data[:code]) - html << '</code></pre></td>' - html << '</tbody></table>' - html << '</div>' unless @nowrap - html - end - - def process_tokens(tokens) - rendered = [] - current_line = '' - - tokens.each do |tok, val| - # In the case of multi-line values (e.g. comments), we need to apply - # styling to each line since span elements are inline. - val.lines.each do |line| - stripped = line.chomp - current_line << span(tok, stripped) - - if line.end_with?("\n") - rendered << current_line - current_line = '' - end - end - end - - # Add leftover text - rendered << current_line if current_line.present? - - num_lines = rendered.size - numbers = (@linenostart..num_lines + @linenostart - 1).to_a - - { numbers: numbers, code: rendered } - end - - def wrap_linenos(numbers) - if @anchorlinenos - numbers.map! do |number| - "<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>" - end - end - numbers.join("\n") - end - - def wrap_lines(lines) - if @lineanchors - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - - if @linenos == 'inline' - "<a name=\"L#{number}\"></a>" \ - "<span class=\"linenos\">#{number}</span>" \ - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - else - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - end - end - elsif @linenos == 'inline' - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - "<span class=\"linenos\">#{number}</span>#{line}" - end - end - - lines.join("\n") - end + def stream(tokens, &b) + is_first = true + token_lines(tokens) do |line| + yield "\n" unless is_first + is_first = false - def span(tok, val) - # http://stackoverflow.com/a/1600584/2587286 - val = CGI.escapeHTML(val) + yield %(<span id="LC#{@line_number}" class="line">) + line.each { |token, value| yield span(token, value.chomp) } + yield %(</span>) - if tok.shortname.empty? - val - else - if @inline_theme - rules = @inline_theme.style_for(tok).rendered_rules - "<span style=\"#{rules.to_a.join(';')}\"#{val}</span>" - else - "<span class=\"#{tok.shortname}\">#{val}</span>" - end + @line_number += 1 end end end diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake new file mode 100644 index 00000000000..afe5d42910c --- /dev/null +++ b/lib/tasks/downtime_check.rake @@ -0,0 +1,12 @@ +desc 'Checks if migrations in a branch require downtime' +task downtime_check: :environment do + if defined?(Gitlab::License) + repo = 'gitlab-ee' + else + repo = 'gitlab-ce' + end + + `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1` + + Rake::Task['gitlab:db:downtime_check'].invoke('FETCH_HEAD') +end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index e930ace1041..993112aee3b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -4,7 +4,7 @@ namespace :gemojione do require 'digest/sha2' require 'json' - dir = Gemojione.index.images_path + dir = Gemojione.images_path digests = [] aliases = Hash.new { |hash, key| hash[key] = [] } aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') @@ -50,9 +50,14 @@ namespace :gemojione do SIZE = 20 RETINA = SIZE * 2 + # Update these values to the width and height of the spritesheet when + # new emoji are added. + SPRITESHEET_WIDTH = 860 + SPRITESHEET_HEIGHT = 840 + Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + FileUtils.cp_r(Gemojione.images_path, tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| @@ -64,7 +69,7 @@ namespace :gemojione do # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" - SpriteFactory.run!(File.join(tmpdir, 'images'), { + SpriteFactory.run!(File.join(tmpdir, 'png'), { output_style: style_path, output_image: "app/assets/images/emoji.png", selector: '.emoji-', @@ -97,7 +102,7 @@ namespace :gemojione do only screen and (min-resolution: 192dpi), only screen and (min-resolution: 2dppx) { background-image: image-url('emoji@2x.png'); - background-size: 840px 820px; + background-size: #{SPRITESHEET_WIDTH}px #{SPRITESHEET_HEIGHT}px; } } CSS @@ -107,7 +112,7 @@ namespace :gemojione do # Now do it again but for Retina Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + FileUtils.cp_r(Gemojione.images_path, tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| @@ -116,7 +121,7 @@ namespace :gemojione do end # Combine the resized assets into a packed sprite and re-generate the SCSS - SpriteFactory.run!(File.join(tmpdir, 'images'), { + SpriteFactory.run!(File.join(tmpdir), { output_image: "app/assets/images/emoji@2x.png", style: false, nocomments: true, diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 5dbf7d61e06..83dd870fa31 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -4,13 +4,13 @@ namespace :gitlab do task all_users_to_all_projects: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) - projects_ids = Project.pluck(:id) + project_ids = Project.pluck(:id) - puts "Importing #{user_ids.size} users into #{projects_ids.size} projects" - ProjectMember.add_users_into_projects(projects_ids, user_ids, ProjectMember::DEVELOPER) + puts "Importing #{user_ids.size} users into #{project_ids.size} projects" + ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) - puts "Importing #{admin_ids.size} admins into #{projects_ids.size} projects" - ProjectMember.add_users_into_projects(projects_ids, admin_ids, ProjectMember::MASTER) + puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects" + ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER) end desc "GitLab | Add a specific user to all projects (as a developer)" @@ -18,7 +18,7 @@ namespace :gitlab do user = User.find_by(email: args.email) project_ids = Project.pluck(:id) puts "Importing #{user.email} users into #{project_ids.size} projects" - ProjectMember.add_users_into_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER) + ProjectMember.add_users_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER) end desc "GitLab | Add all users to all groups (admin users are added as owners)" diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index e9a4e37ec48..5f4a6bbfa35 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -46,7 +46,7 @@ namespace :gitlab do } correct_options = options.map do |name, value| - run(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value end if correct_options.all? @@ -64,7 +64,7 @@ namespace :gitlab do for_more_information( see_installation_guide_section "GitLab" ) - end + end end end @@ -73,7 +73,7 @@ namespace :gitlab do database_config_file = Rails.root.join("config", "database.yml") - if File.exists?(database_config_file) + if File.exist?(database_config_file) puts "yes".color(:green) else puts "no".color(:red) @@ -94,7 +94,7 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") - if File.exists?(gitlab_config_file) + if File.exist?(gitlab_config_file) puts "yes".color(:green) else puts "no".color(:red) @@ -113,7 +113,7 @@ namespace :gitlab do print "GitLab config outdated? ... " gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exists?(gitlab_config_file) + unless File.exist?(gitlab_config_file) puts "can't check because of previous errors".color(:magenta) end @@ -144,7 +144,7 @@ namespace :gitlab do script_path = "/etc/init.d/gitlab" - if File.exists?(script_path) + if File.exist?(script_path) puts "yes".color(:green) else puts "no".color(:red) @@ -169,7 +169,7 @@ namespace :gitlab do recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") script_path = "/etc/init.d/gitlab" - unless File.exists?(script_path) + unless File.exist?(script_path) puts "can't check because of previous errors".color(:magenta) return end @@ -316,7 +316,7 @@ namespace :gitlab do min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " - redis_version = run(%W(redis-cli --version)) + redis_version = run_command(%W(redis-cli --version)) redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) @@ -361,7 +361,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - if File.exists?(repo_base_path) + if File.exist?(repo_base_path) puts "yes".color(:green) else puts "no".color(:red) @@ -385,7 +385,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -408,7 +408,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -438,7 +438,7 @@ namespace :gitlab do Gitlab.config.repositories.storages.each do |name, repo_base_path| print "#{name}... " - unless File.exists?(repo_base_path) + unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) return end @@ -784,7 +784,7 @@ namespace :gitlab do servers.each do |server| puts "Server: #{server}" Gitlab::LDAP::Adapter.open(server) do |adapter| - users = adapter.users(adapter.config.uid, '*', 100) + users = adapter.users(adapter.config.uid, '*', limit) users.each do |user| puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" end @@ -893,7 +893,7 @@ namespace :gitlab do def check_ruby_version required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run(%W(ruby --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version))) print "Ruby version >= #{required_version} ? ... " @@ -910,7 +910,7 @@ namespace :gitlab do def check_git_version required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run(%W(#{Gitlab.config.git.bin_path} --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" print "Git version >= #{required_version} ? ... " diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 7230b9485be..7c96bc864ce 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -25,6 +25,10 @@ namespace :gitlab do desc 'Drop all tables' task :drop_tables => :environment do connection = ActiveRecord::Base.connection + + # If MySQL, turn off foreign key checks + connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? + tables = connection.tables tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run @@ -35,6 +39,9 @@ namespace :gitlab do # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html # Add `IF EXISTS` because cascade could have already deleted a table. tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + + # If MySQL, re-enable foreign key checks + connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql? end desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' @@ -46,5 +53,20 @@ namespace :gitlab do Rake::Task['db:seed_fu'].invoke end end + + desc 'Checks if migrations require downtime or not' + task :downtime_check, [:ref] => :environment do |_, args| + abort 'You must specify a Git reference to compare with' unless args[:ref] + + require 'shellwords' + + ref = Shellwords.escape(args[:ref]) + + migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines. + map { |file| Rails.root.join(file.strip).to_s }. + select { |file| File.file?(file) } + + Gitlab::DowntimeCheck.new.check_and_print(migrations) + end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index fe43d40e6d2..dffea8ed155 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -8,7 +8,7 @@ namespace :gitlab do # check Ruby version ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s) # check Gem version - gem_version = run(%W(gem --version)) + gem_version = run_command(%W(gem --version)) # check Bundler version bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) # check Bundler version @@ -17,7 +17,7 @@ namespace :gitlab do puts "" puts "System information".color(:yellow) puts "System:\t\t#{os_name || "unknown".color(:red)}" - puts "Current User:\t#{run(%W(whoami))}" + puts "Current User:\t#{run_command(%W(whoami))}" puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index c85ebdf8619..bb7eb852f1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -5,7 +5,8 @@ namespace :gitlab do warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - args.with_defaults(tag: 'v' + default_version, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") + default_version_tag = 'v' + default_version + args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") user = Gitlab.config.gitlab.user home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home @@ -15,7 +16,12 @@ namespace :gitlab do target_dir = Gitlab.config.gitlab_shell.path # Clone if needed - unless File.directory?(target_dir) + if File.directory?(target_dir) + Dir.chdir(target_dir) do + system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet)) + system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag})) + end + else system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) end @@ -84,7 +90,7 @@ namespace :gitlab do task build_missing_projects: :environment do Project.find_each(batch_size: 1000) do |project| path_to_repo = project.repository.path_to_repo - if File.exists?(path_to_repo) + if File.exist?(path_to_repo) print '-' else if Gitlab::Shell.new.add_repository(project.repository_storage_path, diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ab96b1d3593..74be413423a 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -23,7 +23,7 @@ namespace :gitlab do # It will primarily use lsb_relase to determine the OS. # It has fallbacks to Debian, SuSE, OS X and systems running systemd. def os_name - os_name = run(%W(lsb_release -irs)) + os_name = run_command(%W(lsb_release -irs)) os_name ||= if File.readable?('/etc/system-release') File.read('/etc/system-release') end @@ -34,7 +34,7 @@ namespace :gitlab do os_name ||= if File.readable?('/etc/SuSE-release') File.read('/etc/SuSE-release') end - os_name ||= if os_x_version = run(%W(sw_vers -productVersion)) + os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) "Mac OS X #{os_x_version}" end os_name ||= if File.readable?('/etc/os-release') @@ -62,10 +62,10 @@ namespace :gitlab do # Returns nil if nothing matched # Returns the MatchData if the pattern matched # - # see also #run + # see also #run_command # see also String#match def run_and_match(command, regexp) - run(command).try(:match, regexp) + run_command(command).try(:match, regexp) end # Runs the given command @@ -74,7 +74,7 @@ namespace :gitlab do # Returns the output of the command otherwise # # see also #run_and_match - def run(command) + def run_command(command) output, _ = Gitlab::Popen.popen(command) output rescue Errno::ENOENT @@ -82,7 +82,7 @@ namespace :gitlab do end def uid_for(user_name) - run(%W(id -u #{user_name})).chomp.to_i + run_command(%W(id -u #{user_name})).chomp.to_i end def gid_for(group_name) @@ -96,7 +96,7 @@ namespace :gitlab do def warn_user_is_not_gitlab unless @warned_user_not_gitlab gitlab_user = Gitlab.config.gitlab.user - current_user = run(%W(whoami)).chomp + current_user = run_command(%W(whoami)).chomp unless current_user == gitlab_user puts " Warning ".color(:black).background(:yellow) puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake new file mode 100644 index 00000000000..84aa2e8507a --- /dev/null +++ b/lib/tasks/gitlab/track_deployment.rake @@ -0,0 +1,9 @@ +namespace :gitlab do + desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' + task track_deployment: :environment do + metric = Gitlab::Metrics::Metric. + new('deployments', version: Gitlab::VERSION) + + Gitlab::Metrics.submit_metrics([metric.to_hash]) + end +end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index f467cc0ee29..49530e7a372 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -26,10 +26,10 @@ namespace :gitlab do namespace_path = ENV['NAMESPACE'] projects = find_projects(namespace_path) - projects_ids = projects.pluck(:id) + project_ids = projects.pluck(:id) puts "Removing webhooks with the url '#{web_hook_url}' ... " - count = WebHook.where(url: web_hook_url, project_id: projects_ids, type: 'ProjectHook').delete_all + count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all puts "#{count} webhooks were removed." end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index da255f5464b..8dbfa7751dc 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -34,21 +34,19 @@ task :spinach do run_spinach_tests(nil) end -def run_command(cmd) +def run_system_command(cmd) system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) end def run_spinach_command(args) - run_command(%w(spinach -r rerun) + args) + run_system_command(%w(spinach -r rerun) + args) end def run_spinach_tests(tags) - #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!') - success = run_spinach_command(%W(--tags #{tags})) 3.times do |_| break if success - break unless File.exists?('tmp/spinach-rerun.txt') + break unless File.exist?('tmp/spinach-rerun.txt') tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) puts '' diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 21c0e5f1d41..d3dcbd2c29b 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -7,5 +7,5 @@ end unless Rails.env.production? desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec] + task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec] end |