diff options
| author | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-07-19 23:33:42 +0200 |
|---|---|---|
| committer | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-07-19 23:33:42 +0200 |
| commit | bf9db45a1dc4d8269fa68dc27dac6667c89ffda8 (patch) | |
| tree | 813d6c8151c156d186cdf4e3420f42260d206434 /lib | |
| parent | 0aedeb5637932fa827e42be7441e9c967049dd1d (diff) | |
| parent | b9ed9d658ad447a64d58b2040849a7cc0e698287 (diff) | |
| download | gitlab-ce-bf9db45a1dc4d8269fa68dc27dac6667c89ffda8.tar.gz | |
Merge remote-tracking branch 'origin/master' into ci-predefined-variables
Diffstat (limited to 'lib')
175 files changed, 3224 insertions, 1440 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 0e7a1cc2623..3d7d67510a8 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -33,14 +33,13 @@ module API mount ::API::Commits mount ::API::DeployKeys mount ::API::Files - mount ::API::Gitignores mount ::API::GroupMembers mount ::API::Groups mount ::API::Internal mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::Licenses + mount ::API::LicenseTemplates mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces @@ -58,6 +57,8 @@ module API mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags + mount ::API::Templates + mount ::API::Todos mount ::API::Triggers mount ::API::Users mount ::API::Variables diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 985590312e3..2efe7e3adf3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -11,7 +11,6 @@ module API [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji # # Parameters: @@ -57,9 +56,9 @@ module API not_found!('Award Emoji') unless can_read_awardable? - award = awardable.award_emoji.new(name: params[:name], user: current_user) + award = awardable.create_award_emoji(params[:name], current_user) - if award.save + if award.persisted? present award, with: Entities::AwardEmoji else not_found!("Award Emoji #{award.errors.messages}") diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 231840148d9..66b853eb342 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -15,7 +15,8 @@ module API # GET /projects/:id/repository/branches get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) - present branches, with: Entities::RepoObject, project: user_project + + present branches, with: Entities::RepoBranch, project: user_project end # Get a single branch @@ -25,10 +26,11 @@ module API # branch (required) - The name of the branch # Example Request: # GET /projects/:id/repository/branches/:branch - get ':id/repository/branches/:branch', requirements: { branch: /.*/ } do + get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } not_found!("Branch") unless @branch - present @branch, with: Entities::RepoObject, project: user_project + + present @branch, with: Entities::RepoBranch, project: user_project end # Protect a single branch @@ -36,19 +38,31 @@ module API # Parameters: # id (required) - The ID of a project # branch (required) - The name of the branch + # developers_can_push (optional) - Flag if developers can push to that branch + # developers_can_merge (optional) - Flag if developers can merge to that branch # Example Request: # PUT /projects/:id/repository/branches/:branch/protect put ':id/repository/branches/:branch/protect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless @branch + not_found!('Branch') unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) - user_project.protected_branches.create(name: @branch.name) unless protected_branch + developers_can_push = to_boolean(params[:developers_can_push]) + developers_can_merge = to_boolean(params[:developers_can_merge]) - present @branch, with: Entities::RepoObject, project: user_project + if protected_branch + protected_branch.developers_can_push = developers_can_push unless developers_can_push.nil? + protected_branch.developers_can_merge = developers_can_merge unless developers_can_merge.nil? + protected_branch.save + else + user_project.protected_branches.create(name: @branch.name, + developers_can_push: developers_can_push || false, + developers_can_merge: developers_can_merge || false) + end + + present @branch, with: Entities::RepoBranch, project: user_project end # Unprotect a single branch @@ -59,8 +73,7 @@ module API # Example Request: # PUT /projects/:id/repository/branches/:branch/unprotect put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) @@ -68,7 +81,7 @@ module API protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch - present @branch, with: Entities::RepoObject, project: user_project + present @branch, with: Entities::RepoBranch, project: user_project end # Create branch @@ -86,7 +99,7 @@ module API if result[:status] == :success present result[:branch], - with: Entities::RepoObject, + with: Entities::RepoBranch, project: user_project else render_api_error!(result[:message], 400) @@ -101,7 +114,7 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.*/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). execute(params[:branch]) diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 979328efe0e..d36047acd1f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -13,7 +13,6 @@ module API # Example Request: # GET /projects/:id/builds get ':id/builds' do - builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -33,10 +32,10 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.pipelines.find_by_sha(params[:sha]) - return not_found! unless commit + return not_found! unless user_project.commit(params[:sha]) - builds = commit.builds.order('id DESC') + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build, diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 323a7086890..acb4812b5cf 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -64,7 +64,7 @@ module API ref = branches.first end - pipeline = @project.ensure_pipeline(commit.sha, ref) + pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) name = params[:name] || params[:context] status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 2e397643ed1..d7e74582459 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -54,10 +54,19 @@ module API class BasicProjectDetails < Grape::Entity expose :id + expose :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace end + class SharedGroup < Grape::Entity + expose :group_id + expose :group_name do |group_link, options| + group_link.group.name + end + expose :group_access, as: :group_access_level + end + class Project < Grape::Entity expose :id, :description, :default_branch, :tag_list expose :public?, as: :public @@ -77,6 +86,9 @@ module API expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds + expose :shared_with_groups do |project, options| + SharedGroup.represent(project.project_group_links.all, options) + end end class ProjectMember < UserBasic @@ -93,6 +105,7 @@ module API class GroupDetail < Group expose :projects, using: Entities::Project + expose :shared_projects, using: Entities::Project end class GroupMember < UserBasic @@ -101,21 +114,23 @@ module API end end - class RepoObject < Grape::Entity + class RepoBranch < Grape::Entity expose :name - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end + expose :commit do |repo_branch, options| + options[:project].repository.commit(repo_branch.target) end - expose :protected do |repo, options| - if options[:project] - options[:project].protected_branch? repo.name - end + expose :protected do |repo_branch, options| + options[:project].protected_branch? repo_branch.name + end + + expose :developers_can_push do |repo_branch, options| + options[:project].developers_can_push_to_protected_branch? repo_branch.name + end + + expose :developers_can_merge do |repo_branch, options| + options[:project].developers_can_merge_to_protected_branch? repo_branch.name end end @@ -174,6 +189,7 @@ module API end expose :user_notes_count expose :upvotes, :downvotes + expose :due_date end class ExternalIssue < Grape::Entity @@ -187,7 +203,6 @@ module API expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels - expose :description expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds @@ -196,6 +211,8 @@ module API merge_request.subscribed?(options[:current_user]) end expose :user_notes_count + expose :should_remove_source_branch?, as: :should_remove_source_branch + expose :force_remove_source_branch?, as: :force_remove_source_branch end class MergeRequestChanges < MergeRequest @@ -240,9 +257,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } - expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } - expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } + expose(:path) { |note| note.diff_file.try(:file_path) if note.diff_note? } + expose(:line) { |note| note.diff_line.try(:new_line) if note.diff_note? } + expose(:line_type) { |note| note.diff_line.try(:type) if note.diff_note? } expose :author, using: Entities::UserBasic expose :created_at end @@ -272,6 +289,31 @@ module API expose :id, :project_id, :group_id, :group_access end + class Todo < Grape::Entity + expose :id + expose :project, using: Entities::BasicProjectDetails + expose :author, using: Entities::UserBasic + expose :action_name + expose :target_type + + expose :target do |todo, options| + Entities.const_get(todo.target_type).represent(todo.target, options) + end + + expose :target_url do |todo, options| + target_type = todo.target_type.underscore + target_url = "namespace_project_#{target_type}_url" + target_anchor = "note_#{todo.note_id}" if todo.note_id? + + Gitlab::Application.routes.url_helpers.public_send(target_url, + todo.project.namespace, todo.project, todo.target, anchor: target_anchor) + end + + expose :body + expose :state + expose :created_at + end + class Namespace < Grape::Entity expose :id, :path, :kind end @@ -376,6 +418,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity @@ -384,27 +427,14 @@ module API end class RepoTag < Grape::Entity - expose :name - expose :message do |repo_obj, _options| - if repo_obj.respond_to?(:message) - repo_obj.message - else - nil - end - end + expose :name, :message - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end + expose :commit do |repo_tag, options| + options[:project].repository.commit(repo_tag.target) end - expose :release, using: Entities::Release do |repo_obj, options| - if options[:project] - options[:project].releases.find_by(tag: repo_obj.name) - end + expose :release, using: Entities::Release do |repo_tag, options| + options[:project].releases.find_by(tag: repo_tag.name) end end @@ -423,6 +453,7 @@ module API class RunnerDetails < Runner expose :tag_list expose :run_untagged + expose :locked expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -444,11 +475,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: User expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: RepoCommit do |repo_obj, _options| - if repo_obj.respond_to?(:commit) - repo_obj.commit.commit_data - end - end + expose :commit, with: RepoCommit expose :runner, with: Runner end @@ -472,11 +499,11 @@ module API expose :content end - class GitignoresList < Grape::Entity + class TemplatesList < Grape::Entity expose :name end - class Gitignore < Grape::Entity + class Template < Grape::Entity expose :name, :content end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb deleted file mode 100644 index 270c9501dd2..00000000000 --- a/lib/api/gitignores.rb +++ /dev/null @@ -1,29 +0,0 @@ -module API - class Gitignores < Grape::API - - # Get the list of the available gitignore templates - # - # Example Request: - # GET /gitignores - get 'gitignores' do - present Gitlab::Gitignore.all, with: Entities::GitignoresList - end - - # Get the text for a specific gitignore - # - # Parameters: - # name (required) - The name of a license - # - # Example Request: - # GET /gitignores/Elixir - # - get 'gitignores/:name' do - required_attributes! [:name] - - gitignore = Gitlab::Gitignore.find(params[:name]) - not_found!('.gitignore') unless gitignore - - present gitignore, with: Entities::Gitignore - end - end -end diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb index ab9b7c602b5..dbe5bb08d3f 100644 --- a/lib/api/group_members.rb +++ b/lib/api/group_members.rb @@ -77,7 +77,7 @@ module API member = group.group_members.find_by(user_id: params[:user_id]) if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}",404) + render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) else member.destroy end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 77e407b54c5..d6e4eb2afd7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -9,6 +9,13 @@ module API [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end + def to_boolean(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + nil + end + def find_user_by_private_token token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) @@ -17,7 +24,7 @@ module API def current_user @current_user ||= (find_user_by_private_token || doorkeeper_guard) - unless @current_user && Gitlab::UserAccess.allowed?(@current_user) + unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 3ac7b50c4ce..959b700de78 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -13,6 +13,7 @@ module API # action - git action (git-upload-pack or git-receive-pack) # ref - branch name # forced_push - forced_push + # protocol - Git access protocol being used, e.g. HTTP or SSH # helpers do @@ -20,11 +21,23 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do - Gitlab::Metrics.action = 'Grape#/internal/allowed' - status 200 actor = @@ -34,24 +47,31 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) + protocol = params[:protocol] access = if wiki? - Gitlab::GitAccessWiki.new(actor, project) + Gitlab::GitAccessWiki.new(actor, project, protocol) else - Gitlab::GitAccess.new(actor, project) + Gitlab::GitAccess.new(actor, project, protocol) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end + end + + response end # diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4c43257c48a..c588103e517 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -59,6 +59,41 @@ module API end end + resource :groups do + # Get a list of group issues + # + # Parameters: + # id (required) - The ID of a group + # state (optional) - Return "opened" or "closed" issues + # labels (optional) - Comma-separated list of label names + # milestone (optional) - Milestone title + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` + # + # Example Requests: + # GET /groups/:id/issues + # GET /groups/:id/issues?state=opened + # GET /groups/:id/issues?state=closed + # GET /groups/:id/issues?labels=foo + # GET /groups/:id/issues?labels=foo,bar + # GET /groups/:id/issues?labels=foo,bar&state=opened + # GET /groups/:id/issues?milestone=1.0.0 + # GET /groups/:id/issues?milestone=1.0.0&state=closed + get ":id/issues" do + group = find_group(params[:id]) + + params[:state] ||= 'opened' + params[:group_id] = group.id + params[:milestone_title] = params.delete(:milestone) + params[:label_name] = params.delete(:labels) + params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + issues = IssuesFinder.new(current_user, params).execute + + present paginate(issues), with: Entities::Issue, current_user: current_user + end + end + resource :projects do # Get a list of project issues # @@ -117,12 +152,13 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # POST /projects/:id/issues - post ":id/issues" do + post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -166,12 +202,13 @@ module API # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # PUT /projects/:id/issues/:issue_id - put ":id/issues/:issue_id" do + put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) diff --git a/lib/api/licenses.rb b/lib/api/license_templates.rb index be0e113fbcb..d0552299ed0 100644 --- a/lib/api/licenses.rb +++ b/lib/api/license_templates.rb @@ -1,6 +1,6 @@ module API - # Licenses API - class Licenses < Grape::API + # License Templates API + class LicenseTemplates < Grape::API PROJECT_TEMPLATE_REGEX = /[\<\{\[] (project|description| diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 0e94efd4acd..4fcdf8968c9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -233,8 +233,8 @@ module API render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? - if params[:sha] && merge_request.source_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.source_sha}", 409) + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) end merge_params = { diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 132043cf3f7..7a0cb7c99f3 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -115,7 +115,6 @@ module API issues = IssuesFinder.new(current_user, finder_params).execute present paginate(issues), with: Entities::Issue, current_user: current_user end - end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index ccca65cbe1c..6bb70bc8bc3 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -28,7 +28,6 @@ module API present @hook, with: Entities::ProjectHook end - # Add hook to project # # Parameters: diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index b703da0557a..6a0b3e7d134 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -4,7 +4,6 @@ module API before { authenticate! } resource :projects do - # Get a project team members # # Parameters: diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5a22d14988f..6d2a6f3946c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,7 +25,11 @@ module API @projects = current_user.authorized_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + if params[:simple] + present @projects, with: Entities::BasicProjectDetails, user: current_user + else + present @projects, with: Entities::ProjectWithAccess, user: current_user + end end # Get an owned projects list for authenticated user @@ -341,7 +345,6 @@ module API else not_found!("Source Project") end - end # Remove a forked_from relationship @@ -418,7 +421,6 @@ module API present paginate(projects), with: Entities::Project end - # Get a users list # # Example Request: diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 4faba9dc87b..ecc8f2fc5a2 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] + attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else @@ -96,9 +96,14 @@ module API runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) - Ci::RunnerProject.create(runner: runner, project: user_project) - present runner, with: Entities::Runner + runner_project = runner.assign_to(user_project) + + if runner_project.persisted? + present runner, with: Entities::Runner + else + conflict!("Runner was already enabled for this project") + end end # Disable project's runner @@ -163,6 +168,7 @@ module API def authenticate_enable_runner!(runner) forbidden!("Runner is shared") if runner.is_shared? + forbidden!("Runner is locked") if runner.locked? return if current_user.is_admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 203f04a6259..fc8598daa32 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -4,7 +4,6 @@ module API before { authenticate! } before { authorize_admin_project } - resource :projects do # Set <service_slug> service for project # diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 3e1ed3fe5c7..7b675e05fbb 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -61,7 +61,7 @@ module API # tag_name (required) - The name of the tag # Example Request: # DELETE /projects/:id/repository/tags/:tag - delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -83,7 +83,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # POST /projects/:id/repository/tags/:tag_name/release - post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = CreateReleaseService.new(user_project, current_user). @@ -104,7 +104,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # PUT /projects/:id/repository/tags/:tag_name/release - put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = UpdateReleaseService.new(user_project, current_user). diff --git a/lib/api/templates.rb b/lib/api/templates.rb new file mode 100644 index 00000000000..18408797756 --- /dev/null +++ b/lib/api/templates.rb @@ -0,0 +1,36 @@ +module API + class Templates < Grape::API + TEMPLATE_TYPES = { + gitignores: Gitlab::Template::Gitignore, + gitlab_ci_ymls: Gitlab::Template::GitlabCiYml + }.freeze + + TEMPLATE_TYPES.each do |template, klass| + # Get the list of the available template + # + # Example Request: + # GET /gitignores + # GET /gitlab_ci_ymls + get template.to_s do + present klass.all, with: Entities::TemplatesList + end + + # Get the text for a specific template + # + # Parameters: + # name (required) - The name of a template + # + # Example Request: + # GET /gitignores/Elixir + # GET /gitlab_ci_ymls/Ruby + get "#{template}/:name" do + required_attributes! [:name] + + new_template = klass.find(params[:name]) + not_found!(template.to_s.singularize) unless new_template + + present new_template, with: Entities::Template + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb new file mode 100644 index 00000000000..26c24c3baff --- /dev/null +++ b/lib/api/todos.rb @@ -0,0 +1,82 @@ +module API + # Todos API + class Todos < Grape::API + before { authenticate! } + + ISSUABLE_TYPES = { + 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'issues' => ->(id) { find_project_issue(id) } + } + + resource :projects do + ISSUABLE_TYPES.each do |type, finder| + type_id_str = "#{type.singularize}_id".to_sym + + # Create a todo on an issuable + # + # Parameters: + # id (required) - The ID of a project + # issuable_id (required) - The ID of an issuable + # Example Request: + # POST /projects/:id/issues/:issuable_id/todo + # POST /projects/:id/merge_requests/:issuable_id/todo + post ":id/#{type}/:#{type_id_str}/todo" do + issuable = instance_exec(params[type_id_str], &finder) + todo = TodoService.new.mark_todo(issuable, current_user).first + + if todo + present todo, with: Entities::Todo, current_user: current_user + else + not_modified! + end + end + end + end + + resource :todos do + helpers do + def find_todos + TodosFinder.new(current_user, params).execute + end + end + + # Get a todo list + # + # Example Request: + # GET /todos + # + get do + todos = find_todos + + present paginate(todos), with: Entities::Todo, current_user: current_user + end + + # Mark a todo as done + # + # Parameters: + # id: (required) - The ID of the todo being marked as done + # + # Example Request: + # DELETE /todos/:id + # + delete ':id' do + todo = current_user.todos.find(params[:id]) + todo.done + + present todo, with: Entities::Todo, current_user: current_user + end + + # Mark all todos as done + # + # Example Request: + # DELETE /todos + # + delete do + todos = find_todos + todos.each(&:done) + + todos.length + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/banzai.rb b/lib/banzai.rb index b467413a7dd..9ebe379f454 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,12 +3,12 @@ module Banzai Renderer.render(text, context) end - def self.render_result(text, context = {}) - Renderer.render_result(text, context) + def self.cache_collection_render(texts_and_contexts) + Renderer.cache_collection_render(texts_and_contexts) end - def self.pre_process(text, context) - Renderer.pre_process(text, context) + def self.render_result(text, context = {}) + Renderer.render_result(text, context) end def self.post_process(html, context) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 81d66271136..d77a5e3ff09 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -160,11 +160,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attribute( - original: link_text || match, - project: project.id, - object_sym => object.id - ) + data = data_attributes_for(link_text || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -183,6 +179,14 @@ module Banzai end end + def data_attributes_for(text, project, object) + data_attribute( + original: text, + project: project.id, + object_sym => object.id + ) + end + def object_link_text_extras(object, matches) extras = [] diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index fac7dad3243..9ed45707515 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -56,6 +56,8 @@ module Banzai # period (e.g., http://localhost:3000/) rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) + return if rinku == html + # Rinku returns a String, so parse it back to a Nokogiri::XML::Document # for further processing. @doc = parse_html(rinku) diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb new file mode 100644 index 00000000000..d2c4b1e4d76 --- /dev/null +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -0,0 +1,71 @@ +module Banzai + module Filter + class BlockquoteFenceFilter < HTML::Pipeline::TextFilter + REGEX = %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?: + # Blockquote: + # >>> + # Anything, including code and HTML blocks + # >>> + + ^>>>\n + (?<quote> + (?: + # Any character that doesn't introduce a code or HTML block + (?! + ^``` + | + ^<[^>]+?>\n + ) + . + | + # A code block + \g<code> + | + # An HTML block + \g<html> + )+? + ) + \n>>>$ + ) + }mx.freeze + + def initialize(text, context = nil, result = nil) + super text, context, result + @text = @text.delete("\r") + end + + def call + @text.gsub(REGEX) do + if $~[:quote] + $~[:quote].gsub(/^/, "> ").gsub(/^> $/, ">") + else + $~[0] + end + end + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d25de900674..ae7d31cf191 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -61,7 +61,7 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end def emoji_pattern @@ -69,7 +69,7 @@ module Banzai end def emoji_filename(name) - "#{Emoji.emoji_filename(name)}.png" + "#{Gitlab::Emoji.emoji_filename(name)}.png" end end end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index ccd106860bd..f0fb6084a35 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -8,6 +8,10 @@ module Banzai # of the anchor, and then replace the img with the link-wrapped version. def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + div = doc.document.create_element( + 'div', + class: 'image-container' + ) link = doc.document.create_element( 'a', @@ -17,7 +21,10 @@ module Banzai ) link.children = img.clone - img.replace(link) + + div.children = link + + img.replace(div) end doc diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2614261f9eb..4042e9a4c25 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -31,10 +31,14 @@ module Banzai projects_per_reference.each do |path, project| issue_ids = references_per_project[path] - next unless project.default_issues_tracker? + if project.default_issues_tracker? + issues = project.issues.where(iid: issue_ids.to_a) + else + issues = issue_ids.map { |id| ExternalIssue.new(id, project) } + end - project.issues.where(iid: issue_ids.to_a).each do |issue| - hash[project][issue.iid] = issue + issues.each do |issue| + hash[project][issue.iid.to_i] = issue end end @@ -42,6 +46,26 @@ module Banzai end end + def object_link_title(object) + if object.is_a?(ExternalIssue) + "Issue in #{object.project.external_issue_tracker.title}" + else + super + end + end + + def data_attributes_for(text, project, object) + if object.is_a?(ExternalIssue) + data_attribute( + project: project.id, + external_issue: object.id, + reference_type: ExternalIssueReferenceFilter.reference_type + ) + else + super + end + end + def find_projects_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e4d3f87d0aa..e258dc8e2bf 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -13,13 +13,13 @@ module Banzai end def self.references_in(text, pattern = Label.reference_pattern) - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ end end def references_in(text, pattern = Label.reference_pattern) - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| label = find_label($~[:project], $~[:label_id], $~[:label_name]) if label @@ -66,6 +66,10 @@ module Banzai LabelsHelper.render_colored_cross_project_label(object) end end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c753a84a20d..c59a80dd1c7 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,40 +7,13 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - nodes = Querying.css(doc, 'a.gfm[data-reference-type]') - visible = nodes_visible_to_user(nodes) - - nodes.each do |node| - unless visible.include?(node) - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) - end - end + Redactor.new(project, current_user).redact([doc]) doc end private - def nodes_visible_to_user(nodes) - per_type = Hash.new { |h, k| h[k] = [] } - visible = Set.new - - nodes.each do |node| - per_type[node.attr('data-reference-type')] << node - end - - per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, current_user) - - visible.merge(parser.nodes_visible_to_user(current_user, nodes)) - end - - visible - end - def current_user context[:current_user] end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d6f34c9cd8..bf058241cda 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -29,7 +29,7 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] = self.class.reference_type + attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index ea21c7b041c..c78da404607 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -14,6 +14,8 @@ module Banzai def call return doc unless linkable_files? + @uri_types = {} + doc.search('a:not(.gfm)').each do |el| process_link_attr el.attribute('href') end @@ -48,7 +50,7 @@ module Banzai uri.path = [ relative_url_root, context[:project].path_with_namespace, - path_type(file_path), + uri_type(file_path), ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') @@ -87,7 +89,7 @@ module Banzai return path unless request_path parts = request_path.split('/') - parts.pop if path_type(request_path) != 'tree' + parts.pop if uri_type(request_path) != :tree while path.start_with?('../') parts.pop @@ -98,45 +100,20 @@ module Banzai end def file_exists?(path) - return false if path.nil? - repository.blob_at(current_sha, path).present? || - repository.tree(current_sha, path).entries.any? - end - - # Get the type of the given path - # - # path - String path to check - # - # Examples: - # - # path_type('doc/README.md') # => 'blob' - # path_type('doc/logo.png') # => 'raw' - # path_type('doc/api') # => 'tree' - # - # Returns a String - def path_type(path) - unescaped_path = Addressable::URI.unescape(path) - - if tree?(unescaped_path) - 'tree' - elsif image?(unescaped_path) - 'raw' - else - 'blob' - end + path.present? && !!uri_type(path) end - def tree?(path) - repository.tree(current_sha, path).entries.any? - end + def uri_type(path) + @uri_types[path] ||= begin + unescaped_path = Addressable::URI.unescape(path) - def image?(path) - repository.blob_at(current_sha, path).try(:image?) + current_commit.uri_type(unescaped_path) + end end - def current_sha - context[:commit].try(:id) || - ref ? repository.commit(ref).try(:sha) : repository.head_commit.sha + def current_commit + @current_commit ||= context[:commit] || + ref ? repository.commit(ref) : repository.head_commit end def relative_url_root @@ -148,7 +125,7 @@ module Banzai end def repository - context[:project].try(:repository) + @repository ||= context[:project].try(:repository) end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 62a79c62e20..91f0159f9a1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -19,24 +19,36 @@ module Banzai language = node.attr('class') code = node.text + css_classes = "code highlight" + + lexer = Rouge::Lexer.find_fancy(language) || Rouge::Lexers::PlainText + formatter = Rouge::Formatters::HTML.new + begin - highlighted = block_code(code, language) + code = formatter.format(lexer.lex(code)) + + css_classes << " js-syntax-highlight #{lexer.tag}" rescue # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. - highlighted = "<pre>#{code}</pre>" end - # Replace the parent `pre` element with the entire highlighted block - node.parent.replace(highlighted) + highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + + # Extracted to a method to measure it + replace_parent_pre_element(node, highlighted) end private + def replace_parent_pre_element(node, highlighted) + # Replace the parent `pre` element with the entire highlighted block + node.parent.replace(highlighted) + end + # Override Rouge::Plugins::Redcarpet#rouge_formatter def rouge_formatter(lexer) - Rouge::Formatters::HTMLGitlab.new( - cssclass: "code highlight js-syntax-highlight #{lexer.tag}") + Rouge::Formatters::HTML.new end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 5b0a6d8541b..e1ca7f4d24b 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -112,7 +112,7 @@ module Banzai data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text) + link_tag(url, data, text, 'All Project and Group Members') end def link_to_namespace(namespace, link_text: nil) @@ -128,7 +128,7 @@ module Banzai data = data_attribute(group: namespace.id) text = link_text || Group.reference_prefix + group - link_tag(url, data, text) + link_tag(url, data, text, namespace.name) end def link_to_user(user, namespace, link_text: nil) @@ -136,11 +136,11 @@ module Banzai data = data_attribute(user: namespace.owner_id) text = link_text || User.reference_prefix + user - link_tag(url, data, text) + link_tag(url, data, text, namespace.owner_name) end - def link_tag(url, data, text) - %(<a href="#{url}" #{data} class="#{link_class}">#{escape_once(text)}</a>) + def link_tag(url, data, text, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) end end end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 1bb6d6bba87..269d5bf74fa 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,7 +8,6 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter - def call return doc unless project_wiki? diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb new file mode 100644 index 00000000000..bab6a9934d1 --- /dev/null +++ b/lib/banzai/note_renderer.rb @@ -0,0 +1,22 @@ +module Banzai + module NoteRenderer + # Renders a collection of Note instances. + # + # notes - The notes to render. + # project - The project to use for rendering/redacting. + # user - The user viewing the notes. + # path - The request path. + # wiki - The project's wiki. + # git_ref - The current Git reference. + def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil) + renderer = ObjectRenderer.new(project, + user, + requested_path: path, + project_wiki: wiki, + ref: git_ref, + pipeline: :note) + + renderer.render(notes, :note) + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb new file mode 100644 index 00000000000..9aef807c152 --- /dev/null +++ b/lib/banzai/object_renderer.rb @@ -0,0 +1,86 @@ +module Banzai + # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # + # Rendered Markdown is stored in an attribute in every object based on the + # name of the attribute containing the Markdown. For example, when the + # attribute `note` is rendered the HTML is stored in `note_html`. + class ObjectRenderer + attr_reader :project, :user + + # Make sure to set the appropriate pipeline in the `raw_context` attribute + # (e.g. `:note` for Note instances). + # + # project - A Project to use for rendering and redacting Markdown. + # user - The user viewing the Markdown/HTML documents, if any. + # context - A Hash containing extra attributes to use in the rendering + # pipeline. + def initialize(project, user = nil, raw_context = {}) + @project = project + @user = user + @raw_context = raw_context + end + + # Renders and redacts an Array of objects. + # + # objects - The objects to render + # attribute - The attribute containing the raw Markdown to render. + # + # Returns the same input objects. + def render(objects, attribute) + documents = render_objects(objects, attribute) + redacted = redact_documents(documents) + + objects.each_with_index do |object, index| + redacted_data = redacted[index] + object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.user_visible_reference_count = redacted_data[:visible_reference_count] + end + end + + # Renders the attribute of every given object. + def render_objects(objects, attribute) + render_attributes(objects, attribute) + end + + # Redacts the list of documents. + # + # Returns an Array containing the redacted documents. + def redact_documents(documents) + redactor = Redactor.new(project, user) + + redactor.redact(documents) + end + + # Returns a Banzai context for the given object and attribute. + def context_for(object, attribute) + context = base_context.merge(cache_key: [object, attribute]) + + if object.respond_to?(:author) + context[:author] = object.author + end + + context + end + + # Renders the attributes of a set of objects. + # + # Returns an Array of `Nokogiri::HTML::Document`. + def render_attributes(objects, attribute) + strings_and_contexts = objects.map do |object| + context = context_for(object, attribute) + + string = object.__send__(attribute) + + { text: string, context: context } + end + + Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| + Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + end + end + + def base_context + @base_context ||= @raw_context.merge(current_user: user, project: project) + end + end +end diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index d47ddfda4be..3c974f73176 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,7 +1,6 @@ module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) - end end end diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb index 50dc978b452..6cf219661d3 100644 --- a/lib/banzai/pipeline/pre_process_pipeline.rb +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -3,7 +3,8 @@ module Banzai class PreProcessPipeline < BasePipeline def self.filters FilterArray[ - Filter::YamlFrontMatterFilter + Filter::YamlFrontMatterFilter, + Filter::BlockquoteFenceFilter, ] end diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb new file mode 100644 index 00000000000..270990e7ab4 --- /dev/null +++ b/lib/banzai/pipeline/relative_link_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class RelativeLinkPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::RelativeLinkFilter + ] + end + end + end +end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb new file mode 100644 index 00000000000..0df3a72d1c4 --- /dev/null +++ b/lib/banzai/redactor.rb @@ -0,0 +1,82 @@ +module Banzai + # Class for removing Markdown references a certain user is not allowed to + # view. + class Redactor + attr_reader :user, :project + + # project - A Project to use for redacting links. + # user - The currently logged in user (if any). + def initialize(project, user = nil) + @project = project + @user = user + end + + # Redacts the references in the given Array of documents. + # + # This method modifies the given documents in-place. + # + # documents - A list of HTML documents containing references to redact. + # + # Returns the documents passed as the first argument. + def redact(documents) + all_document_nodes = document_nodes(documents) + + redact_document_nodes(all_document_nodes) + end + + # Redacts the given node documents + # + # data - An Array of a Hashes mapping an HTML document to nodes to redact. + def redact_document_nodes(all_document_nodes) + all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + visible = nodes_visible_to_user(all_nodes) + metadata = [] + + all_document_nodes.each do |entry| + nodes_for_document = entry[:nodes] + doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + metadata << doc_data + + nodes_for_document.each do |node| + next if visible.include?(node) + + doc_data[:visible_reference_count] -= 1 + # The reference should be replaced by the original text, + # which is not always the same as the rendered text. + text = node.attr('data-original') || node.text + node.replace(text) + end + end + + metadata + end + + # Returns the nodes visible to the current user. + # + # nodes - The input nodes to check. + # + # Returns a new Array containing the visible nodes. + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, user) + + visible.merge(parser.nodes_visible_to_user(user, nodes)) + end + + visible + end + + def document_nodes(documents) + documents.map do |document| + { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') } + end + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 3d7b9c4a024..6cf218aaa0d 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -133,8 +133,9 @@ module Banzai return {} if nodes.empty? ids = unique_attribute_values(nodes, attribute) + rows = collection_objects_for_ids(collection, ids) - collection.where(id: ids).each_with_object({}) do |row, hash| + rows.each_with_object({}) do |row, hash| hash[row.id] = row end end @@ -153,6 +154,31 @@ module Banzai values.to_a end + # Queries the collection for the objects with the given IDs. + # + # If the RequestStore module is enabled this method will only query any + # objects that have not yet been queried. For objects that have already + # been queried the object is returned from the cache. + def collection_objects_for_ids(collection, ids) + if RequestStore.active? + cache = collection_cache[collection_cache_key(collection)] + to_query = ids.map(&:to_i) - cache.keys + + unless to_query.empty? + collection.where(id: to_query).each { |row| cache[row.id] = row } + end + + cache.values + else + collection.where(id: ids) + end + end + + # Returns the cache key to use for a collection. + def collection_cache_key(collection) + collection.respond_to?(:model) ? collection.model : collection + end + # Processes the list of HTML documents and returns an Array containing all # the references. def process(documents) @@ -189,7 +215,7 @@ module Banzai end def find_projects_for_hash_keys(hash) - Project.where(id: hash.keys) + collection_objects_for_ids(Project, hash.keys) end private @@ -199,6 +225,12 @@ module Banzai def lazy(&block) Gitlab::Lazy.new(&block) end + + def collection_cache + RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| + hash[key] = {} + end + end end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index a12b0d19560..863f5725d3b 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -73,7 +73,7 @@ module Banzai def find_users(ids) return [] if ids.empty? - User.where(id: ids).to_a + collection_objects_for_ids(User, ids) end def find_users_for_groups(ids) @@ -85,7 +85,8 @@ module Banzai def find_users_for_projects(ids) return [] if ids.empty? - Project.where(id: ids).flat_map { |p| p.team.members.to_a } + collection_objects_for_ids(Project, ids). + flat_map { |p| p.team.members.to_a } end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c14a9c4c722..910687a7b6a 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -10,7 +10,7 @@ module Banzai # requiring XHTML, such as Atom feeds, need to call `post_process` on the # result, providing the appropriate `pipeline` option. # - # markdown - Markdown String + # text - Markdown String # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String @@ -29,14 +29,62 @@ module Banzai end end - def self.render_result(text, context = {}) - Pipeline[context[:pipeline]].call(text, context) + # Perform multiple render from an Array of Markdown String into an + # Array of HTML-safe String of HTML. + # + # As the rendered Markdown String can be already cached read all the data + # from the cache using Rails.cache.read_multi operation. If the Markdown String + # is not in the cache or it's not cacheable (no cache_key entry is provided in + # the context) the Markdown String is rendered and stored in the cache so the + # next render call gets the rendered HTML-safe String from the cache. + # + # For further explanation see #render method comments. + # + # texts_and_contexts - An Array of Hashes that contains the Markdown String (:text) + # an options passed to our HTML Pipeline (:context) + # + # If on the :context you specify a :cache_key entry will be used to retrieve it + # and cache the result of rendering the Markdown String. + # + # Returns an Array containing HTML-safe String instances. + # + # Example: + # texts_and_contexts + # => [{ text: '### Hello', + # context: { cache_key: [note, :note] } }] + def self.cache_collection_render(texts_and_contexts) + items_collection = texts_and_contexts.each_with_index do |item, index| + context = item[:context] + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + + item[:cache_key] = cache_key if cache_key + end + + cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + + items_in_cache = [] + items_not_in_cache = [] + + unless cacheable_items.empty? + items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) + items_not_in_cache = cacheable_items.reject do |item| + item[:rendered] = items_in_cache[item[:cache_key]] + items_in_cache.key?(item[:cache_key]) + end + end + + (items_not_in_cache + non_cacheable_items).each do |item| + item[:rendered] = render(item[:text], item[:context]) + Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] + end + + items_collection.map { |item| item[:rendered] } end - def self.pre_process(text, context) - pipeline = Pipeline[:pre_process] + def self.render_result(text, context = {}) + text = Pipeline[:pre_process].to_html(text, context) if text - pipeline.to_html(text, context) + Pipeline[context[:pipeline]].call(text, context) end # Perform post-processing on an HTML String @@ -82,5 +130,13 @@ module Banzai return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end + + # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. + # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key + # method. + def self.full_cache_multi_key(cache_key, pipeline_name) + return unless cache_key + Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + end end end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 9f270f7b387..260ac81f5fa 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -195,8 +195,7 @@ module Ci not_found! unless build authenticate_build_token!(build) - build.remove_artifacts_file! - build.remove_artifacts_metadata! + build.erase_artifacts! end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 0c41f22c7c5..bcc82969eb3 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,12 +28,9 @@ module Ci post "register" do required_attributes! [:token] - attributes = { description: params[:description], - tag_list: params[:tag_list] } - - unless params[:run_untagged].nil? - attributes[:run_untagged] = params[:run_untagged] - end + attributes = attributes_for_keys( + [:description, :tag_list, :run_untagged, :locked] + ) runner = if runner_registration_token_valid? diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 5270108ef0f..1d7126a432d 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -13,7 +13,6 @@ module Ci collect end - def push(from, to, format) @labels << from.strftime(format) @total << project.builds. diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index ed86de819eb..83afed9f49f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,9 +2,8 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::ValidationHelpers + include Gitlab::Ci::Config::Node::LegacyValidationHelpers - DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, @@ -14,7 +13,7 @@ module Ci ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :after_script, :image, :services, :path, :cache + attr_reader :path, :cache, :stages def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -22,79 +21,63 @@ module Ci @path = path - initial_parsing + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + initial_parsing validate! rescue Gitlab::Ci::Config::Loader::FormatError => e raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - builds.select do |build| - build[:stage] == stage && - process?(build[:only], build[:except], ref, tag, trigger_request) + def jobs_for_ref(ref, tag = false, trigger_request = nil) + @jobs.select do |_, job| + process?(job[:only], job[:except], ref, tag, trigger_request) end end - def builds - @jobs.map do |name, job| - build_job(name, job) + def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).select do |_, job| + job[:stage] == stage end end - def stages - @stages || DEFAULT_STAGES - end - - def global_variables - @variables - end - - def job_variables(name) - job = @jobs[name.to_sym] - return [] unless job - - job[:variables] || [] + def builds_for_ref(ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).map do |name, _| + build_attributes(name) + end end - private - - def initial_parsing - @after_script = @config[:after_script] - @image = @config[:image] - @services = @config[:services] - @stages = @config[:stages] || @config[:types] - @variables = @config[:variables] || {} - @cache = @config[:cache] - @jobs = {} - - @config.except!(*ALLOWED_YAML_KEYS) - @config.each { |name, param| add_job(name, param) } - - raise ValidationError, "Please define at least one job" if @jobs.none? + def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _| + build_attributes(name) + end end - def add_job(name, job) - return if name.to_s.start_with?('.') - - raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script) - - stage = job[:stage] || job[:type] || DEFAULT_STAGE - @jobs[name] = { stage: stage }.merge(job) + def builds + @jobs.map do |name, _| + build_attributes(name) + end end - def build_job(name, job) + def build_attributes(name) + job = @jobs[name.to_sym] || {} { - stage_idx: stages.index(job[:stage]), + stage_idx: @stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), + ## + # Refactoring note: + # - before script behaves differently than after script + # - after script returns an array of commands + # - before script should be a concatenated command + commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, - only: job[:only], - except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', environment: job[:environment], + yaml_variables: yaml_variables(name), options: { image: job[:image] || @image, services: job[:services] || @services, @@ -106,62 +89,58 @@ module Ci } end - def validate! - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end + private - validate_global! + def initial_parsing + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache - @jobs.each do |name, job| - validate_job!(name, job) - end + @jobs = {} - true - end + @config.except!(*ALLOWED_YAML_KEYS) + @config.each { |name, param| add_job(name, param) } - def validate_global! - unless @after_script.nil? || validate_array_of_strings(@after_script) - raise ValidationError, "after_script should be an array of strings" - end + raise ValidationError, "Please define at least one job" if @jobs.none? + end - unless @image.nil? || @image.is_a?(String) - raise ValidationError, "image should be a string" - end + def add_job(name, job) + return if name.to_s.start_with?('.') - unless @services.nil? || validate_array_of_strings(@services) - raise ValidationError, "services should be an array of strings" - end + raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script) - unless @stages.nil? || validate_array_of_strings(@stages) - raise ValidationError, "stages should be an array of strings" - end + stage = job[:stage] || job[:type] || DEFAULT_STAGE + @jobs[name] = { stage: stage }.merge(job) + end - unless @variables.nil? || validate_variables(@variables) - raise ValidationError, "variables should be a map of key-value strings" + def yaml_variables(name) + variables = global_variables.merge(job_variables(name)) + variables.map do |key, value| + { key: key, value: value, public: true } end + end - validate_global_cache! if @cache + def global_variables + @variables || {} end - def validate_global_cache! - @cache.keys.each do |key| - unless ALLOWED_CACHE_KEYS.include? key - raise ValidationError, "#{name} cache unknown parameter #{key}" - end - end + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job - if @cache[:key] && !validate_string(@cache[:key]) - raise ValidationError, "cache:key parameter should be a string" - end + job[:variables] || {} + end - if @cache[:untracked] && !validate_boolean(@cache[:untracked]) - raise ValidationError, "cache:untracked parameter should be an boolean" + def validate! + @jobs.each do |name, job| + validate_job!(name, job) end - if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) - raise ValidationError, "cache:paths parameter should be an array of strings" - end + true end def validate_job!(name, job) @@ -216,8 +195,8 @@ module Ci raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w[on_success on_failure always]) - raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" + if job[:when] && !job[:when].in?(%w[on_success on_failure always manual]) + raise ValidationError, "#{name} job: when parameter should be on_success, on_failure, always or manual" end if job[:environment] && !validate_environment(job[:environment]) @@ -240,8 +219,8 @@ module Ci end def validate_job_stage!(name, job) - unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" end end @@ -305,12 +284,12 @@ module Ci raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end - stage_index = stages.index(job[:stage]) + stage_index = @stages.index(job[:stage]) job[:dependencies].each do |dependency| raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 42232b7129d..2edddb84fc3 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,62 +7,91 @@ module ContainerRegistry MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + # Taken from: FaradayMiddleware::FollowRedirects + REDIRECT_CODES = Set.new [301, 302, 303, 307] + def initialize(base_uri, options = {}) @base_uri = base_uri - @faraday = Faraday.new(@base_uri) do |conn| - initialize_connection(conn, options) - end + @options = options end def repository_tags(name) - response_body @faraday.get("/v2/#{name}/tags/list") + response_body faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - response_body @faraday.get("/v2/#{name}/manifests/#{reference}") + response_body faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) - response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response = faraday.head("/v2/#{name}/manifests/#{reference}") response.headers['docker-content-digest'] if response.success? end def delete_repository_tag(name, reference) - @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + faraday.delete("/v2/#{name}/manifests/#{reference}").success? end def blob(name, digest, type = nil) - headers = {} - headers['Accept'] = type if type - response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) + type ||= 'application/octet-stream' + response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true end def delete_blob(name, digest) - @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + faraday.delete("/v2/#{name}/blobs/#{digest}").success? end - + private - + def initialize_connection(conn, options) conn.request :json + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + + def accept_manifest(conn) conn.headers['Accept'] = MANIFEST_VERSION conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + end - if options[:user] && options[:password] - conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) - elsif options[:token] - conn.request(:authorization, :bearer, options[:token].to_s) + def response_body(response, allow_redirect: false) + if allow_redirect && REDIRECT_CODES.include?(response.status) + response = redirect_response(response.headers['location']) end - conn.adapter :net_http + response.body if response && response.success? + end + + def redirect_response(location) + return unless location + + # We explicitly remove authorization token + faraday_blob.get(location) do |req| + req['Authorization'] = '' + end end - def response_body(response) - response.body if response.success? + def faraday + @faraday ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + accept_manifest(conn) + end + end + + def faraday_blob + @faraday_blob ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + end end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 7a0929d774e..59040199920 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -3,6 +3,7 @@ module ContainerRegistry attr_reader :repository, :name delegate :registry, :client, to: :repository + delegate :revision, :short_revision, to: :config_blob, allow_nil: true def initialize(repository, name) @repository, @name = repository, name @@ -52,7 +53,7 @@ module ContainerRegistry def config return unless config_blob - @config ||= ContainerRegistry::Config.new(self, config_blob) + @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data end def created_at diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb index 1b80be112a4..cee664b8951 100644 --- a/lib/disable_email_interceptor.rb +++ b/lib/disable_email_interceptor.rb @@ -1,6 +1,5 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class DisableEmailInterceptor - def self.delivering_email(message) message.perform_deliveries = false Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 37f4c34054f..c3064163e07 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -2,6 +2,11 @@ require_dependency 'gitlab/git' module Gitlab def self.com? - Gitlab.config.gitlab.url == 'https://gitlab.com' + # Check `staging?` as well to keep parity with gitlab.com + Gitlab.config.gitlab.url == 'https://gitlab.com' || staging? + end + + def self.staging? + Gitlab.config.gitlab.url == 'https://staging.gitlab.com' end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6d0e30e916f..de41ea415a6 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -5,6 +5,8 @@ # module Gitlab module Access + class AccessDeniedError < StandardError; end + GUEST = 10 REPORTER = 20 DEVELOPER = 30 @@ -12,9 +14,10 @@ module Gitlab OWNER = 50 # Branch protection settings - PROTECTION_NONE = 0 - PROTECTION_DEV_CAN_PUSH = 1 - PROTECTION_FULL = 2 + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 class << self def values @@ -52,6 +55,7 @@ module Gitlab def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, } diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 0b9c2e730f9..1a22ad9acf5 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,7 +4,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index 51b1df9ecbd..39b43ab5489 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -1,24 +1,14 @@ module Gitlab class AwardEmoji CATEGORIES = { - other: "Other", objects: "Objects", - places: "Places", - travel_places: "Travel", - emoticons: "Emoticons", - objects_symbols: "Symbols", + travel: "Travel", + symbols: "Symbols", nature: "Nature", - celebration: "Celebration", people: "People", activity: "Activity", flags: "Flags", - food_drink: "Food" - }.with_indifferent_access - - CATEGORY_ALIASES = { - symbols: "objects_symbols", - foods: "food_drink", - travel: "travel_places" + food: "Food" }.with_indifferent_access def self.normalize_emoji_name(name) @@ -35,7 +25,7 @@ module Gitlab # Skip Fitzpatrick(tone) modifiers next if data["category"] == "modifier" - category = CATEGORY_ALIASES[data["category"]] || data["category"] + category = data["category"] @emoji_by_category[category] << data end @@ -57,17 +47,26 @@ module Gitlab def self.aliases @aliases ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) - JSON.parse(File.read(json_path)) - end + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + JSON.parse(File.read(json_path)) + end end # Returns an Array of Emoji names and their asset URLs. def self.urls @urls ||= begin path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + # Construct the full asset path ourselves because + # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds + # of entries since it has to do a lot of extra work (e.g. regexps). prefix = Gitlab::Application.config.assets.prefix digest = Gitlab::Application.config.assets.digest + base = + if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root + Gitlab::Application.config.relative_url_root + else + '' + end JSON.parse(File.read(path)).map do |hash| if digest @@ -76,7 +75,7 @@ module Gitlab fname = hash['unicode'] end - { name: hash['name'], path: "#{prefix}/#{fname}.png" } + { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 7e3f5abba62..ab94abeda77 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -8,7 +8,6 @@ module Grack end class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env def call(env) @@ -22,7 +21,7 @@ module Grack # Need this if under RELATIVE_URL_ROOT unless Gitlab.config.gitlab.relative_url_root.empty? # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'') + @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') else @env['PATH_INFO'] = @request.path end @@ -31,7 +30,7 @@ module Grack auth! - lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call return lfs_response unless lfs_response.nil? if @user.nil? && !@ci @@ -64,7 +63,7 @@ module Grack def ci_request?(login, password) matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) - if project && matched_login.present? && git_cmd == 'git-upload-pack' + if project && matched_login.present? underscored_service = matched_login['s'].underscore if underscored_service == 'gitlab_ci' diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 997a22779a0..d62bc50ce78 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -41,7 +41,8 @@ module Gitlab def highlighted_lines @blob.load_all_data!(repository) - @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + @highlighted_lines ||= + Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines end def project diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb new file mode 100644 index 00000000000..5551fac4b8b --- /dev/null +++ b/lib/gitlab/checks/change_access.rb @@ -0,0 +1,96 @@ +module Gitlab + module Checks + class ChangeAccess + attr_reader :user_access, :project + + def initialize(change, user_access:, project:) + @oldrev, @newrev, @ref = change.split(' ') + @branch_name = branch_name(@ref) + @user_access = user_access + @project = project + end + + def exec + error = protected_branch_checks || tag_checks || push_checks + + if error + GitAccessStatus.new(false, error) + else + GitAccessStatus.new(true) + end + end + + protected + + def protected_branch_checks + return unless project.protected_branch?(@branch_name) + + if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + return "You are not allowed to force push code to a protected branch on this project." + elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + return "You are not allowed to delete protected branches from this project." + end + + if matching_merge_request? + if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) + return + else + "You are not allowed to merge code into protected branches on this project." + end + else + if user_access.can_push_to_branch?(@branch_name) + return + else + "You are not allowed to push code to protected branches on this project." + end + end + end + + def tag_checks + tag_ref = tag_name(@ref) + + if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) + "You are not allowed to change existing tags on this project." + end + end + + def push_checks + if user_access.cannot_do_action?(:push_code) + "You are not allowed to push code to this project." + end + end + + private + + def protected_tag?(tag_name) + project.repository.tag_exists?(tag_name) + end + + def forced_push? + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def matching_merge_request? + Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? + end + + def branch_name(ref) + ref = @ref.to_s + if Gitlab::Git.branch_ref?(ref) + Gitlab::Git.ref_name(ref) + else + nil + end + end + + def tag_name(ref) + ref = @ref.to_s + if Gitlab::Git.tag_ref?(ref) + Gitlab::Git.ref_name(ref) + else + nil + end + end + end + end +end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb new file mode 100644 index 00000000000..5fe86553bd0 --- /dev/null +++ b/lib/gitlab/checks/force_push.rb @@ -0,0 +1,17 @@ +module Gitlab + module Checks + class ForcePush + def self.force_push?(project, oldrev, newrev) + return false if project.empty_repo? + + # Created or deleted branch + if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) + false + else + missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) + missed_ref.present? + end + end + end + end +end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb new file mode 100644 index 00000000000..849848515da --- /dev/null +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -0,0 +1,18 @@ +module Gitlab + module Checks + class MatchingMergeRequest + def initialize(newrev, branch_name, project) + @newrev = newrev + @branch_name = branch_name + @project = project + end + + def match? + @project.merge_requests + .with_state(:locked) + .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) + .exists? + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index b48d3592f16..e6cc1529760 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,12 +4,11 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - delegate :valid?, :errors, to: :@global - ## # Temporary delegations that should be removed after refactoring # - delegate :before_script, to: :@global + delegate :before_script, :image, :services, :after_script, :variables, + :stages, :cache, to: :@global def initialize(config) @config = Loader.new(config).load! @@ -18,6 +17,14 @@ module Gitlab @global.process! end + def valid? + @global.valid? + end + + def errors + @global.errors + end + def to_hash @config end diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/node/boolean.rb new file mode 100644 index 00000000000..84b03ee7832 --- /dev/null +++ b/lib/gitlab/ci/config/node/boolean.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a boolean value. + # + class Boolean < Entry + include Validatable + + validations do + validates :config, boolean: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/node/cache.rb new file mode 100644 index 00000000000..cdf8ba2e35d --- /dev/null +++ b/lib/gitlab/ci/config/node/cache.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a cache configuration + # + class Cache < Entry + include Configurable + + node :key, Node::Key, + description: 'Cache key used to define a cache affinity.' + + node :untracked, Node::Boolean, + description: 'Cache all untracked files.' + + node :paths, Node::Paths, + description: 'Specify which paths should be cached across builds.' + + validations do + validates :config, allowed_keys: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index d60f87f3f94..37936fc8242 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -15,43 +15,49 @@ module Gitlab # module Configurable extend ActiveSupport::Concern + include Validatable - def allowed_nodes - self.class.allowed_nodes || {} + included do + validations do + validates :config, type: Hash + end end private - def prevalidate! - unless @value.is_a?(Hash) - @errors << 'should be a configuration entry with hash value' - end - end - def create_node(key, factory) - factory.with(value: @value[key]) - factory.nullify! unless @value.has_key?(key) + factory.with(value: @config[key], key: key, parent: self) + factory.create! end class_methods do - def allowed_nodes - Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + def nodes + Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end private - def allow_node(symbol, entry_class, metadata) + def node(symbol, entry_class, metadata) factory = Node::Factory.new(entry_class) .with(description: metadata[:description]) - define_method(symbol) do - raise Entry::InvalidError unless valid? + (@nodes ||= {}).merge!(symbol.to_sym => factory) + end - @nodes[symbol].try(:value) - end + def helpers(*nodes) + nodes.each do |symbol| + define_method("#{symbol}_defined?") do + @nodes[symbol].try(:defined?) + end - (@allowed_nodes ||= {}).merge!(symbol => factory) + define_method("#{symbol}_value") do + raise Entry::InvalidError unless valid? + @nodes[symbol].try(:value) + end + + alias_method symbol.to_sym, "#{symbol}_value".to_sym + end end end end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 52758a962f3..9e79e170a4f 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -8,14 +8,14 @@ module Gitlab class Entry class InvalidError < StandardError; end - attr_accessor :description + attr_reader :config + attr_accessor :key, :parent, :description - def initialize(value) - @value = value + def initialize(config) + @config = config @nodes = {} - @errors = [] - - prevalidate! + @validator = self.class.validator.new(self) + @validator.validate end def process! @@ -23,50 +23,65 @@ module Gitlab return unless valid? compose! - - nodes.each(&:process!) - nodes.each(&:validate!) + process_nodes! end def nodes @nodes.values end - def valid? - errors.none? + def leaf? + self.class.nodes.none? end - def leaf? - allowed_nodes.none? + def ancestors + @parent ? @parent.ancestors + [@parent] : [] + end + + def valid? + errors.none? end def errors - @errors + nodes.map(&:errors).flatten + @validator.messages + nodes.flat_map(&:errors) end - def allowed_nodes - {} + def value + if leaf? + @config + else + defined = @nodes.select { |_key, value| value.defined? } + Hash[defined.map { |key, node| [key, node.value] }] + end end - def validate! - raise NotImplementedError + def defined? + true end - def value - raise NotImplementedError + def self.default end - private + def self.nodes + {} + end - def prevalidate! + def self.validator + Validator end + private + def compose! - allowed_nodes.each do |key, essence| + self.class.nodes.each do |key, essence| @nodes[key] = create_node(key, essence) end end + def process_nodes! + nodes.each(&:process!) + end + def create_node(key, essence) raise NotImplementedError end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 787ca006f5a..5919a283283 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -5,13 +5,11 @@ module Gitlab ## # Factory class responsible for fabricating node entry objects. # - # It uses Fluent Interface pattern to set all necessary attributes. - # class Factory class InvalidFactory < StandardError; end - def initialize(entry_class) - @entry_class = entry_class + def initialize(node) + @node = node @attributes = {} end @@ -20,18 +18,29 @@ module Gitlab self end - def nullify! - @entry_class = Node::Null - self - end - def create! raise InvalidFactory unless @attributes.has_key?(:value) - @entry_class.new(@attributes[:value]).tap do |entry| + fabricate.tap do |entry| + entry.key = @attributes[:key] + entry.parent = @attributes[:parent] entry.description = @attributes[:description] end end + + private + + def fabricate + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @attributes[:value].nil? + Node::Undefined.new(@node) + else + @node.new(@attributes[:value]) + end + end end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index 044603423d5..f92e1eccbcf 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -9,8 +9,36 @@ module Gitlab class Global < Entry include Configurable - allow_node :before_script, Script, + node :before_script, Node::Script, description: 'Script that will be executed before each job.' + + node :image, Node::Image, + description: 'Docker image that will be used to execute jobs.' + + node :services, Node::Services, + description: 'Docker images that will be linked to the container.' + + node :after_script, Node::Script, + description: 'Script that will be executed after each job.' + + node :variables, Node::Variables, + description: 'Environment variables that will be used.' + + node :stages, Node::Stages, + description: 'Configuration of stages for this pipeline.' + + node :types, Node::Stages, + description: 'Deprecated: stages for this pipeline.' + + node :cache, Node::Cache, + description: 'Configure caching between build jobs.' + + helpers :before_script, :image, :services, :after_script, + :variables, :stages, :types, :cache + + def stages + stages_defined? ? stages_value : types_value + end end end end diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/node/image.rb new file mode 100644 index 00000000000..5d3c7c5eab0 --- /dev/null +++ b/lib/gitlab/ci/config/node/image.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a Docker image. + # + class Image < Entry + include Validatable + + validations do + validates :config, type: String + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/node/key.rb new file mode 100644 index 00000000000..f8b461ca098 --- /dev/null +++ b/lib/gitlab/ci/config/node/key.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a key. + # + class Key < Entry + include Validatable + + validations do + validates :config, key: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 72f648975dc..4d9a508796a 100644 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -2,7 +2,7 @@ module Gitlab module Ci class Config module Node - module ValidationHelpers + module LegacyValidationHelpers private def validate_duration(value) diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 4f590f6bec8..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents a configuration entry that is not being used - # in configuration file. - # - # This implements Null Object pattern. - # - class Null < Entry - def value - nil - end - - def validate! - nil - end - - def method_missing(*) - nil - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/node/paths.rb new file mode 100644 index 00000000000..3c6d3a52966 --- /dev/null +++ b/lib/gitlab/ci/config/node/paths.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an array of paths. + # + class Paths < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb index 5072bf0db7d..39328f0fade 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/node/script.rb @@ -5,22 +5,11 @@ module Gitlab ## # Entry that represents a script. # - # Each element in the value array is a command that will be executed - # by GitLab Runner. Currently we concatenate these commands with - # new line character as a separator, what is compatible with - # implementation in Runner. - # class Script < Entry - include ValidationHelpers - - def value - @value.join("\n") - end + include Validatable - def validate! - unless validate_array_of_strings(@value) - @errors << 'before_script should be an array of strings' - end + validations do + validates :config, array_of_strings: true end end end diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/node/services.rb new file mode 100644 index 00000000000..481e2b66adc --- /dev/null +++ b/lib/gitlab/ci/config/node/services.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration of Docker services. + # + class Services < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/node/stages.rb new file mode 100644 index 00000000000..b1fe45357ff --- /dev/null +++ b/lib/gitlab/ci/config/node/stages.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration for pipeline stages. + # + class Stages < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + + def self.default + %w[build test deploy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb new file mode 100644 index 00000000000..699605e1e3a --- /dev/null +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an undefined entry node. + # + # It takes original entry class as configuration and returns default + # value of original entry as self value. + # + # + class Undefined < Entry + include Validatable + + validations do + validates :config, type: Class + end + + def value + @config.default + end + + def defined? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb new file mode 100644 index 00000000000..f6e2896dfb2 --- /dev/null +++ b/lib/gitlab/ci/config/node/validatable.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + module Validatable + extend ActiveSupport::Concern + + class_methods do + def validator + validator = Class.new(Node::Validator) + + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end + + validator + end + + private + + def validations(&block) + (@validations ||= []).append(block) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb new file mode 100644 index 00000000000..758a6cf4356 --- /dev/null +++ b/lib/gitlab/ci/config/node/validator.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + class Config + module Node + class Validator < SimpleDelegator + include ActiveModel::Validations + include Node::Validators + + def initialize(node) + super(node) + @node = node + end + + def messages + errors.full_messages.map do |error| + "#{location} #{error}".downcase + end + end + + def self.name + 'Validator' + end + + def unknown_keys + return [] unless config.is_a?(Hash) + + config.keys - @node.class.nodes.keys + end + + private + + def location + predecessors = ancestors.map(&:key).compact + current = key || @node.class.name.demodulize.underscore + predecessors.append(current).join(':') + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb new file mode 100644 index 00000000000..7b2f57990b5 --- /dev/null +++ b/lib/gitlab/ci/config/node/validators.rb @@ -0,0 +1,70 @@ +module Gitlab + module Ci + class Config + module Node + module Validators + class AllowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + if record.unknown_keys.any? + unknown_list = record.unknown_keys.join(', ') + record.errors.add(:config, + "contains unknown keys: #{unknown_list}") + end + end + end + + class ArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_array_of_strings(value) + record.errors.add(attribute, 'should be an array of strings') + end + end + end + + class BooleanValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_boolean(value) + record.errors.add(attribute, 'should be a boolean value') + end + end + end + + class KeyValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_string(value) + record.errors.add(attribute, 'should be a string or symbol') + end + end + end + + class TypeValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + type = options[:with] + raise unless type.is_a?(Class) + + unless value.is_a?(type) + record.errors.add(attribute, "should be a #{type.name}") + end + end + end + + class VariablesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs') + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/node/variables.rb new file mode 100644 index 00000000000..5f813f81f55 --- /dev/null +++ b/lib/gitlab/ci/config/node/variables.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents environment variables. + # + class Variables < Entry + include Validatable + + validations do + validates :config, variables: true + end + + def self.default + {} + end + end + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 28c34429c1f..ffc1814b29d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,10 +9,14 @@ module Gitlab end def ensure_application_settings! - settings = ::ApplicationSetting.cached + if connect_to_db? + begin + settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + settings = ::ApplicationSetting.last + end - if !settings && connect_to_db? - settings = ::ApplicationSetting.current settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end @@ -44,6 +48,7 @@ module Gitlab akismet_enabled: false, repository_checks_enabled: true, container_registry_token_expire_delay: 5, + user_default_external: false, ) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dec20d8659b..927f9dad20b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -20,11 +20,19 @@ module Gitlab if Database.postgresql? options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout end add_index(table_name, column_name, options) end + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. (PostgreSQL only) + def disable_statement_timeout + ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -133,6 +141,8 @@ module Gitlab 'in the body of your migration class' end + disable_statement_timeout + transaction do add_column(table, column, type, default: nil) diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb new file mode 100644 index 00000000000..8406ca4269c --- /dev/null +++ b/lib/gitlab/diff/diff_refs.rb @@ -0,0 +1,36 @@ +module Gitlab + module Diff + class DiffRefs + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(base_sha:, start_sha: base_sha, head_sha:) + @base_sha = base_sha + @start_sha = start_sha + @head_sha = head_sha + end + + def ==(other) + other.is_a?(self.class) && + base_sha == other.base_sha && + start_sha == other.start_sha && + head_sha == other.head_sha + end + + # There is only one case in which we will have `start_sha` and `head_sha`, + # but not `base_sha`, which is when a diff is generated between an + # orphaned branch and another branch, which means there _is_ no base, but + # we're still able to highlight it, and to create diff notes, which are + # the primary things `DiffRefs` are used for. + # `DiffRefs` are "complete" when they have `start_sha` and `head_sha`, + # because `base_sha` can always be derived from this, to return an actual + # sha, or `nil`. + # We have `base_sha` directly available on `DiffRefs` because it's faster# + # than having to look it up in the repo every time. + def complete? + start_sha && head_sha + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2e85cabf72..b09ca1fb8b0 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,47 +1,83 @@ module Gitlab module Diff class File - attr_reader :diff, :diff_refs + attr_reader :diff, :repository, :diff_refs delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, to: :diff, prefix: false + :old_path, :new_path, :a_mode, :b_mode, + :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false - def initialize(diff, diff_refs) + def initialize(diff, repository:, diff_refs: nil) @diff = diff + @repository = repository @diff_refs = diff_refs end + def position(line) + return unless diff_refs + + Position.new( + old_path: old_path, + new_path: new_path, + old_line: line.old_line, + new_line: line.new_line, + diff_refs: diff_refs + ) + end + + def line_code(line) + return if line.meta? + + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def line_for_line_code(code) + diff_lines.find { |line| line_code(line) == code } + end + + def line_for_position(pos) + diff_lines.find { |line| position(line) == pos } + end + + def position_for_line_code(code) + line = line_for_line_code(code) + position(line) if line + end + + def line_code_for_position(pos) + line = line_for_position(pos) + line_code(line) if line + end + + def content_commit + return unless diff_refs + + repository.commit(deleted_file ? old_ref : new_ref) + end + def old_ref - diff_refs[0] if diff_refs + diff_refs.try(:base_sha) end def new_ref - diff_refs[1] if diff_refs + diff_refs.try(:head_sha) end - # Array of Gitlab::DIff::Line objects + # Array of Gitlab::Diff::Line objects def diff_lines - @lines ||= parser.parse(raw_diff.each_line).to_a - end - - def too_large? - diff.too_large? + @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a end def highlighted_diff_lines - Gitlab::Diff::Highlight.new(self).highlight + @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end def parallel_diff_lines - Gitlab::Diff::ParallelDiff.new(self).parallelize + @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end def mode_changed? - !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) - end - - def parser - Gitlab::Diff::Parser.new + a_mode && b_mode && a_mode != b_mode end def raw_diff @@ -53,17 +89,15 @@ module Gitlab end def prev_line(index) - if index > 0 - diff_lines[index - 1] - end + diff_lines[index - 1] if index > 0 + end + + def paths + [old_path, new_path].compact end def file_path - if diff.new_path.present? - diff.new_path - elsif diff.old_path.present? - diff.old_path - end + new_path.presence || old_path end def added_lines @@ -73,6 +107,21 @@ module Gitlab def removed_lines diff_lines.count(&:removed?) end + + def old_blob(commit = content_commit) + return unless commit + + parent_id = commit.parent_id + return unless parent_id + + repository.blob_at(parent_id, old_path) + end + + def blob(commit = content_commit) + return unless commit + + repository.blob_at(commit.id, file_path) + end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 9429b3ff88d..649a265a02c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,11 +1,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines + attr_reader :diff_file, :diff_lines, :raw_lines, :repository delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - def initialize(diff_lines) + def initialize(diff_lines, repository: nil) + @repository = repository + if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @diff_lines = @diff_file.diff_lines @@ -19,7 +21,7 @@ module Gitlab @diff_lines.map.with_index do |diff_line, i| diff_line = diff_line.dup # ignore highlighting for "match" lines - next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + next diff_line if diff_line.meta? rich_line = highlight_line(diff_line) || diff_line.text @@ -40,12 +42,12 @@ module Gitlab line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' - case diff_line.type - when 'new', nil - rich_line = new_lines[diff_line.new_pos - 1] - when 'old' - rich_line = old_lines[diff_line.old_pos - 1] - end + rich_line = + if diff_line.unchanged? || diff_line.added? + new_lines[diff_line.new_pos - 1] + elsif diff_line.removed? + old_lines[diff_line.old_pos - 1] + end # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. @@ -58,19 +60,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) - end - - def processing_args(version) - ref = send("diff_#{version}_ref") - path = send("diff_#{version}_path") - - [ref.project.repository, ref.id, path] + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) end end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 789c14518b0..28ad637fda4 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,16 +1,30 @@ module Gitlab module Diff class InlineDiff + # Regex to find a run of deleted lines followed by the same number of added lines + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze + attr_accessor :old_line, :new_line, :offset def self.for_lines(lines) - local_edit_indexes = self.find_local_edits(lines) + changed_line_pairs = self.find_changed_line_pairs(lines) inline_diffs = [] - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 + changed_line_pairs.each do |old_index, new_index| old_line = lines[old_index] new_line = lines[new_index] @@ -51,18 +65,28 @@ module Gitlab private - def self.find_local_edits(lines) - line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - joined_line_prefixes = " #{line_prefixes.join} " - - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + # Finds pairs of old/new line pairs that represent the same line that changed + def self.find_changed_line_pairs(lines) + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + + changed_line_pairs = [] + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + changed_line_pairs << [i, i + changed_line_count] + end end - local_edit_indexes + changed_line_pairs end def longest_common_prefix(a, b) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 03730b435ad..c6189d660c2 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -9,6 +9,18 @@ module Gitlab @old_pos, @new_pos = old_pos, new_pos end + def old_line + old_pos unless added? || meta? + end + + def new_line + new_pos unless removed? || meta? + end + + def unchanged? + type.nil? + end + def added? type == 'new' end @@ -16,6 +28,10 @@ module Gitlab def removed? type == 'old' end + + def meta? + type == 'match' || type == 'nonewline' + end end end end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb new file mode 100644 index 00000000000..576a761423e --- /dev/null +++ b/lib/gitlab/diff/line_mapper.rb @@ -0,0 +1,64 @@ +# When provided a diff for a specific file, maps old line numbers to new line +# numbers and back, to find out where a specific line in a file was moved by the +# changes. +module Gitlab + module Diff + class LineMapper + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + # Find new line number for old line number. + def old_to_new(old_line) + map_line_number(old_line, from: :old_line, to: :new_line) + end + + # Find old line number for new line number. + def new_to_old(new_line) + map_line_number(new_line, from: :new_line, to: :old_line) + end + + private + + def diff_lines + @diff_lines ||= @diff_file.diff_lines + end + + # Find old/new line number based on its old/new counterpart line number. + def map_line_number(from_line, from:, to:) + # If no diff file could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_file + + # To find the mapped line number for the specified line number, + # we need to find: + # - The diff line with that exact line number, if it is in the diff context + # - The first diff line with a higher line number, if it falls between diff contexts + # - The last known diff line, if it falls after the last diff context + diff_line = diff_lines.find do |diff_line| + diff_from_line = diff_line.send(from) + diff_from_line && diff_from_line >= from_line + end + diff_line ||= diff_lines.last + + # If no diff line could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_line + + diff_from_line = diff_line.send(from) + diff_to_line = diff_line.send(to) + + # If the line was removed, there is no mapped line number. + return unless diff_to_line + + # Because we may not have the diff line with the exact line number + # we were looking for, we need to adjust the mapped line number. + distance = diff_from_line - from_line + + diff_to_line - distance + end + end + end +end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 74f9b3c050a..b069afdd28c 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -8,111 +8,96 @@ module Gitlab end def parallelize - lines = [] - skip_next = false + i = 0 + free_right_index = nil + + lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - full_line = line.text - type = line.type - line_code = generate_line_code(diff_file.file_path, line) - line_new = line.new_pos - line_old = line.old_pos + line_code = diff_file.line_code(line) + position = diff_file.position(line) - next_line = diff_file.next_line(line.index) - - if next_line - next_line = highlighted_diff_lines[next_line.index] - next_line_code = generate_line_code(diff_file.file_path, next_line) - next_type = next_line.type - next_line = next_line.text - end - - case type + case line.type when 'match', nil # line in the right panel is the same as in the left one lines << { left: { - type: type, - number: line_old, - text: full_line, + type: line.type, + number: line.old_pos, + text: line.text, line_code: line_code, + position: position }, right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code + type: line.type, + number: line.new_pos, + text: line.text, + line_code: line_code, + position: position } } + + free_right_index = nil + i += 1 when 'old' - case next_type - when 'new' - # Left side has text removed, right side has text added - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - }, - right: { - type: next_type, - number: line_new, - text: next_line, - line_code: next_line_code - } - } - skip_next = true - when 'old', 'nonewline', nil - # Left side has text removed, right side doesn't have any change - # No next line code, no new line number, no new line text - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - }, - right: { - type: next_type, - number: nil, - text: "", - line_code: nil - } + lines << { + left: { + type: line.type, + number: line.old_pos, + text: line.text, + line_code: line_code, + position: position + }, + right: { + type: nil, + number: nil, + text: "", + line_code: line_code, + position: position } - end + } + + # Once we come upon a new line it can be put on the right of this old line + free_right_index ||= i + i += 1 when 'new' - if skip_next - # Change has been already included in previous line so no need to do it again - skip_next = false - next + data = { + type: line.type, + number: line.new_pos, + text: line.text, + line_code: line_code, + position: position + } + + if free_right_index + # If an old line came before this without a line on the right, this + # line can be put to the right of it. + lines[free_right_index][:right] = data + + # If there are any other old lines on the left that don't yet have + # a new counterpart on the right, update the free_right_index + next_free_right_index = free_right_index + 1 + free_right_index = next_free_right_index < i ? next_free_right_index : nil else - # Change is only on the right side, left side has no change lines << { left: { type: nil, number: nil, text: "", line_code: line_code, + position: position }, - right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code - } + right: data } + + free_right_index = nil + i += 1 end end end - lines - end - private - - def generate_line_code(file_path, line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + lines end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 522dd2b9428..59a2367b65d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -40,7 +40,6 @@ module Gitlab line_obj_index += 1 end - case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb new file mode 100644 index 00000000000..989fff8918e --- /dev/null +++ b/lib/gitlab/diff/position.rb @@ -0,0 +1,155 @@ +# Defines a specific location, identified by paths and line numbers, +# within a specific diff, identified by start, head and base commit ids. +module Gitlab + module Diff + class Position + attr_reader :old_path + attr_reader :new_path + attr_reader :old_line + attr_reader :new_line + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(attrs = {}) + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + if attrs[:diff_refs] + @base_sha = attrs[:diff_refs].base_sha + @start_sha = attrs[:diff_refs].start_sha + @head_sha = attrs[:diff_refs].head_sha + else + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + end + + # `Gitlab::Diff::Position` objects are stored as serialized attributes in + # `DiffNote`, which use YAML to encode and decode objects. + # `#init_with` and `#encode_with` can be used to customize the en/decoding + # behavior. In this case, we override these to prevent memoized instance + # variables like `@diff_file` and `@diff_line` from being serialized. + def init_with(coder) + initialize(coder['attributes']) + + self + end + + def encode_with(coder) + coder['attributes'] = self.to_h + end + + def key + @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + end + + def ==(other) + other.is_a?(self.class) && key == other.key + end + + def to_h + { + old_path: old_path, + new_path: new_path, + old_line: old_line, + new_line: new_line, + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha + } + end + + def inspect + %(#<#{self.class}:#{object_id} #{to_h}>) + end + + def complete? + file_path.present? && + (old_line || new_line) && + diff_refs.complete? + end + + def to_json + JSON.generate(self.to_h) + end + + def type + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def unchanged? + type.nil? + end + + def added? + type == 'new' + end + + def removed? + type == 'old' + end + + def paths + [old_path, new_path].compact.uniq + end + + def file_path + new_path.presence || old_path + end + + def diff_refs + @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) + end + + def diff_file(repository) + @diff_file ||= begin + if RequestStore.active? + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + RequestStore.fetch(key) { find_diff_file(repository) } + else + find_diff_file(repository) + end + end + end + + def diff_line(repository) + @diff_line ||= diff_file(repository).line_for_position(self) + end + + def line_code(repository) + @line_code ||= diff_file(repository).line_code_for_position(self) + end + + private + + def find_diff_file(repository) + diffs = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ).diffs(paths: paths) + + diff = diffs.first + return unless diff + + Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb new file mode 100644 index 00000000000..4d04f867268 --- /dev/null +++ b/lib/gitlab/diff/position_tracer.rb @@ -0,0 +1,168 @@ +# Finds the diff position in the new diff that corresponds to the same location +# specified by the provided position in the old diff. +module Gitlab + module Diff + class PositionTracer + attr_accessor :repository + attr_accessor :old_diff_refs + attr_accessor :new_diff_refs + attr_accessor :paths + + def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) + @repository = repository + @old_diff_refs = old_diff_refs + @new_diff_refs = new_diff_refs + @paths = paths + end + + def trace(old_position) + return unless old_diff_refs.complete? && new_diff_refs.complete? + return unless old_position.diff_refs == old_diff_refs + + # Suppose we have an MR with source branch `feature` and target branch `master`. + # When the MR was created, the head of `master` was commit A, and the + # head of `feature` was commit B, resulting in the original diff A->B. + # Since creation, `master` was updated to C. + # Now `feature` is being updated to D, and the newly generated MR diff is C->D. + # It is possible that C and D are direct decendants of A and B respectively, + # but this isn't necessarily the case as rebases and merges come into play. + # + # Suppose we have a diff note on the original diff A->B. Now that the MR + # is updated, we need to find out what line in C->D corresponds to the + # line the note was originally created on, so that we can update the diff note's + # records and continue to display it in the right place in the diffs. + # If we cannot find this line in the new diff, this means the diff note is now + # outdated, and we will display that fact to the user. + # + # In the new diff, the file the diff note was originally created on may + # have been renamed, deleted or even created, if the file existed in A and B, + # but was removed in C, and restored in D. + # + # Every diff note stores a Position object that defines a specific location, + # identified by paths and line numbers, within a specific diff, identified + # by start, head and base commit ids. + # + # For diff notes for diff A->B, the position looks like this: + # Position + # base_sha - ID of commit A + # head_sha - ID of commit B + # old_path - path as of A (nil if file was newly created) + # new_path - path as of B (nil if file was deleted) + # old_line - line number as of A (nil if file was newly created) + # new_line - line number as of B (nil if file was deleted) + # + # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # but need to find the paths and line numbers as of C and D. + # + # If the file was unchanged or newly created in A->B, the path as of D can be found + # by generating diff B->D ("head to head"), finding the diff file with + # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. + # The path as of C can be found by taking diff C->D, finding the diff file + # with that same `new_path` and taking `diff_file.old_path`. + # The line number as of D can be found by using the LineMapper on diff B->D + # and providing the line number as of B. + # The line number as of C can be found by using the LineMapper on diff C->D + # and providing the line number as of D. + # + # If the file was deleted in A->B, the path as of C can be found + # by generating diff A->C ("base to base"), finding the diff file with + # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. + # The path as of D can be found by taking diff C->D, finding the diff file + # with that same `old_path` and taking `diff_file.new_path`. + # The line number as of C can be found by using the LineMapper on diff A->C + # and providing the line number as of A. + # The line number as of D can be found by using the LineMapper on diff C->D + # and providing the line number as of C. + + results = nil + results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? + results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? + + return unless results + + file_diff, old_line, new_line = results + + Position.new( + old_path: file_diff.old_path, + new_path: file_diff.new_path, + head_sha: new_diff_refs.head_sha, + start_sha: new_diff_refs.start_sha, + base_sha: new_diff_refs.base_sha, + old_line: old_line, + new_line: new_line + ) + end + + private + + def trace_added_line(old_position) + file_path = old_position.new_path + + return unless diff_head_to_head + + file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_head_to_head.new_path if file_head_to_head + + new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) + + return unless new_line + + file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } + return unless file_diff + + old_line = LineMapper.new(file_diff).new_to_old(new_line) + + [file_diff, old_line, new_line] + end + + def trace_removed_line(old_position) + file_path = old_position.old_path + + return unless diff_base_to_base + + file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_base_to_base.old_path if file_base_to_base + + old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + + return unless old_line + + file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } + return unless file_diff + + new_line = LineMapper.new(file_diff).old_to_new(old_line) + + [file_diff, old_line, new_line] + end + + def diff_base_to_base + @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + end + + def diff_head_to_head + @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + end + + def new_diffs + @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + end + + def diff_files(start_sha, head_sha, use_base: false) + base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + + diffs = self.repository.raw_repository.diff( + use_base ? base_sha : start_sha, + head_sha, + {}, + *paths + ) + + diffs.decorate! do |diff| + Gitlab::Diff::File.new(diff, repository: self.repository) + end + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index e2fee6b9f3e..97701b0cd42 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -33,11 +33,15 @@ module Gitlab end def commits - @commits ||= (Commit.decorate(compare.commits, project) if compare) + return unless compare + + @commits ||= Commit.decorate(compare.commits, project) end def diffs - @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) + return unless compare + + @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository) end def diffs_count diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 97ef9851d71..1c671a7487b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -104,15 +104,7 @@ module Gitlab end def create_note(reply) - Notes::CreateService.new( - sent_notification.project, - sent_notification.recipient, - note: reply, - noteable_type: sent_notification.noteable_type, - noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code - ).execute + sent_notification.create_note(reply) end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb new file mode 100644 index 00000000000..b63213ae208 --- /dev/null +++ b/lib/gitlab/emoji.rb @@ -0,0 +1,21 @@ +module Gitlab + module Emoji + extend self + + def emojis + Gemojione.index.instance_variable_get(:@emoji_by_name) + end + + def emojis_by_moji + Gemojione.index.instance_variable_get(:@emoji_by_moji) + end + + def emojis_names + emojis.keys.sort + end + + def emoji_filename(name) + emojis[name]["unicode"] + end + end +end diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb deleted file mode 100644 index 93c6a5bb7f5..00000000000 --- a/lib/gitlab/force_push_check.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Gitlab - class ForcePushCheck - def self.force_push?(project, oldrev, newrev) - return false if project.empty_repo? - - # Created or deleted branch - if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - false - else - missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) - missed_refs.split("\n").size > 0 - end - end - end -end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 07b856ca64c..9b681e636c7 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,6 +1,7 @@ module Gitlab module Git class Hook + GL_PROTOCOL = 'web'.freeze attr_reader :name, :repo_path, :path def initialize(name, repo_path) @@ -14,7 +15,7 @@ module Gitlab end def trigger(gl_id, oldrev, newrev, ref) - return true unless exists? + return [true, nil] unless exists? case name when "pre-receive", "post-receive" @@ -29,19 +30,20 @@ module Gitlab def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") - # function will return true if succesful exit_status = false + exit_message = nil vars = { 'GL_ID' => gl_id, - 'PWD' => repo_path + 'PWD' => repo_path, + 'GL_PROTOCOL' => GL_PROTOCOL } options = { chdir: repo_path } - Open3.popen2(vars, path, options) do |stdin, _, wait_thr| + Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| exit_status = true stdin.sync = true @@ -60,17 +62,24 @@ module Gitlab unless wait_thr.value == 0 exit_status = false + exit_message = retrieve_error_message(stderr, stdout) end end - exit_status + [exit_status, exit_message] end def call_update_hook(gl_id, oldrev, newrev, ref) Dir.chdir(repo_path) do - system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + [status.success?, stderr.presence || stdout] end end + + def retrieve_error_message(stderr, stdout) + err_message = stderr.gets + err_message.blank? ? stdout.gets : err_message + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index d2a0e316cbe..8e8f39d9cb2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -1,63 +1,31 @@ +# Check a user's access to perform a git action. All public methods in this +# class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project + attr_reader :actor, :project, :protocol, :user_access - def initialize(actor, project) + def initialize(actor, project, protocol) @actor = actor @project = project - end - - def user - return @user if defined?(@user) - - @user = - case actor - when User - actor - when DeployKey - nil - when Key - actor.user - end - end - - def deploy_key - actor if actor.is_a?(DeployKey) - end - - def can_push_to_branch?(ref) - return false unless user - - if project.protected_branch?(ref) && !project.developers_can_push_to_protected_branch?(ref) - user.can?(:push_code_to_protected_branches, project) - else - user.can?(:push_code, project) - end - end - - def can_read_project? - if user - user.can?(:read_project, project) - elsif deploy_key - deploy_key.projects.include?(project) - else - false - end + @protocol = protocol + @user_access = UserAccess.new(user, project: project) end def check(cmd, changes = nil) + return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? + unless actor return build_status_object(false, "No user or key was provided.") end - if user && !user_allowed? + if user && !user_access.allowed? return build_status_object(false, "Your account has been blocked.") end - unless project && can_read_project? + unless project && (user_access.can_read_project? || deploy_key_can_read_project?) return build_status_object(false, 'The project you were looking for could not be found.') end @@ -92,7 +60,7 @@ module Gitlab end def user_download_access_check - unless user.can?(:download_code, project) + unless user_access.can_do_action?(:download_code) return build_status_object(false, "You are not allowed to download code from this project.") end @@ -122,92 +90,49 @@ module Gitlab build_status_object(true) end - def can_user_do_action?(action) - @permission_cache ||= {} - @permission_cache[action] ||= user.can?(action, project) - end - def change_access_check(change) - oldrev, newrev, ref = change.split(' ') - - action = - if project.protected_branch?(branch_name(ref)) - protected_branch_action(oldrev, newrev, branch_name(ref)) - elsif (tag_ref = tag_name(ref)) && protected_tag?(tag_ref) - # Prevent any changes to existing git tag unless user has permissions - :admin_project - else - :push_code - end - - unless can_user_do_action?(action) - status = - case action - when :force_push_code_to_protected_branches - build_status_object(false, "You are not allowed to force push code to a protected branch on this project.") - when :remove_protected_branches - build_status_object(false, "You are not allowed to deleted protected branches from this project.") - when :push_code_to_protected_branches - build_status_object(false, "You are not allowed to push code to protected branches on this project.") - when :admin_project - build_status_object(false, "You are not allowed to change existing tags on this project.") - else # :push_code - build_status_object(false, "You are not allowed to push code to this project.") - end - return status - end - - build_status_object(true) + Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec end - def forced_push?(oldrev, newrev) - Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) + def protocol_allowed? + Gitlab::ProtocolAccess.allowed?(protocol) end private - def protected_branch_action(oldrev, newrev, branch_name) - # we dont allow force push to protected branch - if forced_push?(oldrev, newrev) - :force_push_code_to_protected_branches - elsif Gitlab::Git.blank_ref?(newrev) - # and we dont allow remove of protected branch - :remove_protected_branches - elsif project.developers_can_push_to_protected_branch?(branch_name) - :push_code - else - :push_code_to_protected_branches - end - end - - def protected_tag?(tag_name) - project.repository.tag_exists?(tag_name) - end - - def user_allowed? - Gitlab::UserAccess.allowed?(user) + def matching_merge_request?(newrev, branch_name) + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end - def branch_name(ref) - ref = ref.to_s - if Gitlab::Git.branch_ref?(ref) - Gitlab::Git.ref_name(ref) - else - nil - end + def deploy_key + actor if actor.is_a?(DeployKey) end - def tag_name(ref) - ref = ref.to_s - if Gitlab::Git.tag_ref?(ref) - Gitlab::Git.ref_name(ref) + def deploy_key_can_read_project? + if deploy_key + return true if project.public? + deploy_key.projects.include?(project) else - nil + false end end protected + def user + return @user if defined?(@user) + + @user = + case actor + when User + actor + when DeployKey + nil + when Key + actor.user + end + end + def build_status_object(status, message = '') GitAccessStatus.new(status, message) end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 8672cbc0ec4..f71d3575909 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,7 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess def change_access_check(change) - if user.can?(:create_wiki, project) + if user_access.can_do_action?(:create_wiki) build_status_object(true) else build_status_object(false, "You are not allowed to write to this project's wiki.") diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index a15fc84b418..7d2d545b84e 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -4,7 +4,7 @@ module Gitlab delegate :repo, :sha, :ref, to: :raw_data def exists? - project.repository.branch_exists?(ref) + branch_exists? && commit_exists? end def name @@ -15,11 +15,15 @@ module Gitlab repo.present? end - def valid? - repo.present? + private + + def branch_exists? + project.repository.branch_exists?(ref) end - private + def commit_exists? + project.repository.commit(sha).present? + end def short_id sha.to_s[0..7] diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..084e514492c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,15 +69,30 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? end def rate_limit_exceed? - rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS end def rate_limit_sleep_time diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..3932fcb1eda 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -131,8 +131,10 @@ module Gitlab def clean_up_restored_branches(branches) branches.each do |name, _| client.delete_ref(repo, "heads/#{name}") - project.repository.rm_branch(project.creator, name) + project.repository.delete_branch(name) rescue Rugged::ReferenceError end + + project.repository.after_remove_branch end def apply_labels(issuable) @@ -167,7 +169,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 498b00cb658..a4ea2210abd 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -11,10 +11,10 @@ module Gitlab description: description, source_project: source_branch_project, source_branch: source_branch_name, - head_source_sha: source_branch_sha, + source_branch_sha: source_branch_sha, target_project: target_branch_project, target_branch: target_branch_name, - base_target_sha: target_branch_sha, + target_branch_sha: target_branch_sha, state: state, milestone: milestone, author_id: author_id, diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb deleted file mode 100644 index f46b43b61a4..00000000000 --- a/lib/gitlab/gitignore.rb +++ /dev/null @@ -1,56 +0,0 @@ -module Gitlab - class Gitignore - FILTER_REGEX = /\.gitignore\z/.freeze - - def initialize(path) - @path = path - end - - def name - File.basename(@path, '.gitignore') - end - - def content - File.read(@path) - end - - class << self - def all - languages_frameworks + global - end - - def find(key) - file_name = "#{key}.gitignore" - - directory = select_directory(file_name) - directory ? new(File.join(directory, file_name)) : nil - end - - def global - files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } - end - - def languages_frameworks - files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } - end - - private - - def select_directory(file_name) - [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } - end - - def global_dir - File.join(gitignore_dir, 'Global') - end - - def gitignore_dir - Rails.root.join('vendor/gitignore') - end - - def files_for_folder(dir) - Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } - end - end - end -end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 3f76ec97977..46d40f75be6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -15,31 +15,35 @@ module Gitlab end def execute - project_identifier = CGI.escape(project.import_source) - - # Issues && Comments - issues = client.issues(project_identifier) - - issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"]) - body += issue["description"] - - comments = client.issue_comments(project_identifier, issue["id"]) - - if comments.any? - body += @formatter.comments_header + ActiveRecord::Base.no_touching do + project_identifier = CGI.escape(project.import_source) + + # Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] + + comments = client.issue_comments(project_identifier, issue["id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + end + + project.issues.create!( + iid: issue["iid"], + description: body, + title: issue["title"], + state: issue["state"], + updated_at: issue["updated_at"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) end - - comments.each do |comment| - body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) - end - - project.issues.create!( - description: body, - title: issue["title"], - state: issue["state"], - author_id: gl_user_id(project, issue["author"]["id"]) - ) end true diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f751a3a12fd..c5a11148d33 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,10 +3,9 @@ module Gitlab def add_gon_variables gon.api_version = API::API.version gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = help_shortcuts_path + gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 2122339d2db..3caf9036459 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= (@commits.size.to_f / @duration).round(1) + @commit_per_day ||= @commits.size / (@duration + 1) end def collect_data diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 280120b0f9e..9360afedfcb 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap). + def self.highlight(blob_name, blob_content, repository: nil, plain: false) + new(blob_name, blob_content, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -10,35 +10,45 @@ module Gitlab return [] unless blob blob.load_all_data!(repository) - highlight(file_name, blob.data).lines.map!(&:html_safe) + highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - def initialize(blob_name, blob_content, nowrap: true) - @formatter = rouge_formatter(nowrap: nowrap) - @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + def initialize(blob_name, blob_content, repository: nil) + @formatter = Rouge::Formatters::HTMLGitlab.new + @repository = repository + @blob_name = blob_name + @blob_content = blob_content end def highlight(text, continue: true, plain: false) if plain - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + hl_lexer = Rouge::Lexers::PlainText + continue = false else - @formatter.format(@lexer.lex(text, continue: continue)).html_safe + hl_lexer = self.lexer end + + @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe rescue @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end + def lexer + @lexer ||= custom_language || begin + Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.sort_by(&:tag).first + end + end + private - def rouge_formatter(options = {}) - options = options.reverse_merge( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) + def custom_language + language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') + + return nil unless language_name - Rouge::Formatters::HTMLGitlab.new(options) + Rouge::Lexer.find_fancy(language_name) end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 99cf85d9a3b..d6d14bd98a0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.0' + VERSION = '0.1.2' + FILENAME_LIMIT = 50 def export_path(relative_path:) File.join(storage_path, relative_path) @@ -28,6 +29,12 @@ module Gitlab 'VERSION' end + def export_filename(project:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}" + + "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" + end + def version VERSION end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb new file mode 100644 index 00000000000..352539eb594 --- /dev/null +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -0,0 +1,31 @@ +module Gitlab + module ImportExport + class AvatarRestorer + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true unless avatar_export_file + + @project.avatar = File.open(avatar_export_file) + @project.save! + rescue => e + @shared.error(e) + false + end + + private + + def avatar_export_file + @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + end + + def avatar_export_path + File.join(@shared.export_path, 'avatar') + end + end + end +end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb new file mode 100644 index 00000000000..998c21e2586 --- /dev/null +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -0,0 +1,31 @@ +module Gitlab + module ImportExport + class AvatarSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless @project.avatar.exists? + + copy_files(avatar_path, avatar_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def avatar_export_path + File.join(@shared.export_path, 'avatar', @project.avatar_identifier) + end + + def avatar_path + @project.avatar.path + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 78664f076eb..5dd0e34c18e 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -28,13 +28,23 @@ module Gitlab end def execute(cmd) - _output, status = Gitlab::Popen.popen(cmd) + output, status = Gitlab::Popen.popen(cmd) + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status.zero? status.zero? end def git_bin_path Gitlab.config.git.bin_path end + + def copy_files(source, destination) + # if we are copying files, create the destination folder + destination_folder = File.file?(source) ? File.dirname(destination) : destination + + FileUtils.mkdir_p(destination_folder) + FileUtils.copy_entry(source, destination) + true + end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 0e70d9282d5..82d1e1805c5 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -23,7 +23,11 @@ module Gitlab private def decompress_archive - untar_zxf(archive: @archive_file, dir: @shared.export_path) + result = untar_zxf(archive: @archive_file, dir: @shared.export_path) + + raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + + true end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 164ab6238c4..15afe8174a4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,24 +1,29 @@ # Model relationships to be included in the project import/export project_tree: - issues: + - :events - notes: - :author + - :author + - :events - :labels - - :milestones + - milestones: + - :events - snippets: - notes: :author - :releases - - :events - project_members: - :user - merge_requests: - notes: - :author + - :author + - :events - :merge_request_diff + - :events - pipelines: - notes: - :author + - :author + - :events - :statuses - :variables - :triggers @@ -48,7 +53,11 @@ included_attributes: excluded_attributes: snippets: - :expired_at + merge_request_diff: + - :st_diffs methods: statuses: - - :type
\ No newline at end of file + - :type + merge_request_diff: + - :utf8_st_diffs
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index d209e04f7be..e9ee47fc090 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Importer - def initialize(project) @archive_file = project.import_source @current_user = project.creator @@ -10,17 +9,22 @@ module Gitlab end def execute - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, - shared: @shared) - if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end + + remove_import_file end private + def import_file + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + end + def check_version! Gitlab::ImportExport::VersionChecker.check!(shared: @shared) end @@ -31,6 +35,10 @@ module Gitlab project: @project) end + def avatar_restorer + Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: @shared, @@ -40,8 +48,7 @@ module Gitlab def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project), - wiki: true) + project: ProjectWiki.new(project_tree.restored_project)) end def uploads_restorer @@ -59,6 +66,10 @@ module Gitlab def wiki_repo_path File.join(@shared.export_path, 'project.wiki.bundle') end + + def remove_import_file + FileUtils.rm_rf(@archive_file) + end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c569a35a48b..b459054c198 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids def initialize(exported_members:, user:, project:) diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index 89388d1984b..77bb3ca6581 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectCreator - def initialize(namespace_id, current_user, file, project_path) @namespace_id = namespace_id @current_user = current_user diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dd71b92c522..051110c23cf 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user @@ -13,7 +12,10 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') - create_relations + + ActiveRecord::Base.no_touching do + create_relations + end rescue => e @shared.error(e) false @@ -70,10 +72,19 @@ module Gitlab # Example: # +relation_key+ issues, loops through the list of *issues* and for each individual # issue, finds any subrelations such as notes, creates them and assign them back to the hash + # + # Recursively calls this method if the sub-relation is a hash containing more sub-relations def create_sub_relations(relation, tree_hash) relation_key = relation.keys.first.to_s + return if tree_hash[relation_key].blank? + tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author + + create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 19defd8f03a..15f5dd31035 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Reader - attr_reader :tree def initialize(shared:) @@ -55,7 +54,6 @@ module Gitlab @json_config_hash end - # If the model is a hash, process the sub_models, which could also be hashes # If there is a list, add to an existing array, otherwise use hash syntax # +current_key+ main model that will be a key in the hash diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b872780f20a..e41c7e6bf4f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class RelationFactory - OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', @@ -12,6 +11,8 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze + def self.create(*args) new(*args).create end @@ -31,6 +32,8 @@ module Gitlab update_user_references update_project_references reset_ci_tokens if @relation_name == 'Ci::Trigger' + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] + set_st_diffs if @relation_name == :merge_request_diff generate_imported_object end @@ -70,7 +73,7 @@ module Gitlab end def generate_imported_object - if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') imported_object do |object| object.trace = trace @@ -85,7 +88,7 @@ module Gitlab project_id = @relation_hash.delete('project_id') # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['project_id'] = project_id @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] @@ -109,7 +112,7 @@ module Gitlab end def imported_object - imported_object = relation_class.new(@relation_hash) + imported_object = relation_class.new(parsed_relation_hash) yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object @@ -123,6 +126,14 @@ module Gitlab def admin_user? @user.is_admin? end + + def parsed_relation_hash + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end + + def set_st_diffs + @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 546dae4d122..f84de652a57 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,15 +3,14 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:, wiki: false) + def initialize(project:, shared:, path_to_bundle:) @project = project @path_to_bundle = path_to_bundle @shared = shared - @wiki = wiki end def restore - return wiki? unless File.exist?(@path_to_bundle) + return true unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) @@ -30,10 +29,6 @@ module Gitlab def path_to_repo @project.repository.path_to_repo end - - def wiki? - @wiki - end end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index cce43fe994b..331e14021e6 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -11,7 +11,7 @@ module Gitlab end def save - return false if @project.empty_repo? + return true if @project.empty_repo? # it's ok to have no repo @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index f38229c6c59..6130c124dd1 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -7,7 +7,8 @@ module Gitlab new(*args).save end - def initialize(shared:) + def initialize(project:, shared:) + @project = project @shared = shared end @@ -17,6 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") archive_file else + @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) false end rescue => e @@ -35,7 +37,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project)) end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 6aff05b886a..5d6de8bc475 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts def initialize(opts) diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 7292e9d9712..62a2553675c 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,6 +1,7 @@ module Gitlab module ImportExport class UploadsSaver + include Gitlab::ImportExport::CommandLineUtil def initialize(project:, shared:) @project = project @@ -18,12 +19,6 @@ module Gitlab private - def copy_files(source, destination) - FileUtils.mkdir_p(destination) - FileUtils.copy_entry(source, destination) - true - end - def uploads_export_path File.join(@shared.export_path, 'uploads') end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index cf5c62c5e3c..abfc694b879 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionChecker - def self.check!(*args) new(*args).check! end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index f7f73dc9343..9b642d740b7 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionSaver - def initialize(shared:) @shared = shared end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1eedae39f8a..6107420e4dd 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab def save @wiki = ProjectWiki.new(@project) return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 948d43582cf..59a05411fe9 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -24,8 +24,6 @@ module Gitlab 'GitLab export' => 'gitlab_project' } end - end - end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index 8684b4636ea..b75ae512d92 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -39,7 +39,7 @@ module Gitlab # OpenSSH 6.8 introduces a new default output format for fingerprints. # Check the version and decide which command to use. - version_output, version_status = popen(%W(ssh -V)) + version_output, version_status = popen(%w(ssh -V)) return false unless version_status.zero? version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 9d9617761b3..a1ee1aa81ff 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -1,11 +1,11 @@ module Gitlab module Lfs class Response - - def initialize(project, user, request) + def initialize(project, user, ci, request) @origin_project = project @project = storage_project(project) @user = user + @ci = ci @env = request.env @request = request end @@ -47,6 +47,8 @@ module Gitlab end def render_storage_upload_store_response(oid, size, tmp_file_name) + return render_forbidden unless tmp_file_name + render_response_to_push do render_lfs_upload_ok(oid, size, tmp_file_name) end @@ -189,7 +191,7 @@ module Gitlab return render_not_enabled unless Gitlab.config.lfs.enabled unless @project.public? - return render_unauthorized unless @user + return render_unauthorized unless @user || @ci return render_forbidden unless user_can_fetch? end @@ -210,7 +212,7 @@ module Gitlab def user_can_fetch? # Check user access against the project they used to initiate the pull - @user.can?(:download_code, @origin_project) + @ci || @user.can?(:download_code, @origin_project) end def user_can_push? diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 78d02891102..f2a76a56b8f 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -1,9 +1,12 @@ module Gitlab module Lfs class Router - def initialize(project, user, request) + attr_reader :project, :user, :ci, :request + + def initialize(project, user, ci, request) @project = project @user = user + @ci = ci @env = request.env @request = request end @@ -71,8 +74,6 @@ module Gitlab lfs.render_storage_upload_authorize_response(oid, size) else tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) - return nil unless tmp_file_name - lfs.render_storage_upload_store_response(oid, size, tmp_file_name) end end @@ -80,7 +81,7 @@ module Gitlab def lfs return unless @project - Gitlab::Lfs::Response.new(@project, @user, @request) + Gitlab::Lfs::Response.new(@project, @user, @ci, @request) end def sanitize_tmp_filename(name) diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index faf0d9b6318..c048fe20ba7 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -18,12 +18,12 @@ module Gitlab # Measures the real and CPU execution time of the supplied block. def measure - start_real = Time.now + start_real = System.monotonic_time start_cpu = System.cpu_time retval = yield - @real_time += (Time.now - start_real) * 1000.0 - @cpu_time += System.cpu_time.to_f - start_cpu + @real_time += System.monotonic_time - start_real + @cpu_time += System.cpu_time - start_cpu @call_count += 1 retval diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 1cd1ca30f70..f23d67e1e38 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,16 +4,15 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags, :created_at + attr_reader :series, :values, :tags # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. def initialize(series, values, tags = {}) - @values = values - @series = series - @tags = tags - @created_at = Time.now.utc + @values = values + @series = series + @tags = tags end # Returns a Hash in a format that can be directly written to InfluxDB. @@ -27,20 +26,20 @@ module Gitlab # # Due to the way InfluxDB is set up there's no solution to this problem, # all we can do is lower the amount of collisions. We do this by using - # Time#to_f which returns the seconds as a Float providing greater - # accuracy. We then add a small random value that is large enough to - # distinguish most timestamps but small enough to not alter the amount - # of seconds. + # System.real_time which returns the nanoseconds as a Float providing + # greater accuracy. We then add a small random value that is large + # enough to distinguish most timestamps but small enough to not alter + # the timestamp significantly. # # See https://gitlab.com/gitlab-com/operations/issues/175 for more # information. - time = @created_at.to_f + rand(JITTER_RANGE) + time = System.real_time(:nanosecond) + rand(JITTER_RANGE) { series: @series, tags: @tags, values: @values, - timestamp: (time * 1_000_000_000).to_i + timestamp: time.to_i } end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index fd98aa3412e..a1240fd33ee 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -8,6 +8,8 @@ module Gitlab trans = Transaction.new("#{worker.class.name}#perform") begin + # Old gitlad-shell messages don't provide enqueued_at/created_at attributes + trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } ensure trans.finish diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 8e345e8ae4a..aaed2184f44 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -2,11 +2,21 @@ module Gitlab module Metrics module Subscribers # Class for tracking the total time spent in Rails cache calls + # http://guides.rubyonrails.org/active_support_instrumentation.html class RailsCache < ActiveSupport::Subscriber attach_to :active_support def cache_read(event) increment(:cache_read, event.duration) + + return unless current_transaction + return if event.payload[:super_operation] == :fetch + + if event.payload[:hit] + current_transaction.increment(:cache_read_hit_count, 1) + else + current_transaction.increment(:cache_read_miss_count, 1) + end end def cache_write(event) @@ -21,6 +31,18 @@ module Gitlab increment(:cache_exists, event.duration) end + def cache_fetch_hit(event) + return unless current_transaction + + current_transaction.increment(:cache_read_hit_count, 1) + end + + def cache_generate(event) + return unless current_transaction + + current_transaction.increment(:cache_read_miss_count, 1) + end + def increment(key, duration) return unless current_transaction diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a7d183b2f94..82c18bb108b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,29 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f end else def self.cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f end end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float. + def self.real_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + end + + # Returns the current monotonic clock time in a given precision. + # + # Returns the time as a Float. + def self.monotonic_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 4bc5081aa03..bded245da43 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -30,7 +30,7 @@ module Gitlab end def duration - @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + @finished_at ? (@finished_at - @started_at) : 0.0 end def allocated_memory @@ -41,12 +41,12 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = Time.now + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = Time.now + @finished_at = System.monotonic_time Thread.current[THREAD_KEY] = nil end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 36e5c2670bb..7d6911a1ab3 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -66,7 +66,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 78f3ecb4cb4..0a91d3918d5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -56,8 +56,6 @@ module Gitlab if external_provider? && @user @user.external = true - elsif @user - @user.external = false end @user @@ -74,7 +72,7 @@ module Gitlab if user # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) + user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) else log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." user = find_by_uid_and_provider diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 746ec283330..4e2f8ed5587 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -1,7 +1,6 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup - # Public: Converts the provided markup into HTML. # # input - the source text in a markup format diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb new file mode 100644 index 00000000000..21aefc884be --- /dev/null +++ b/lib/gitlab/protocol_access.rb @@ -0,0 +1,13 @@ +module Gitlab + module ProtocolAccess + def self.allowed?(protocol) + if protocol == 'web' + true + elsif current_application_settings.enabled_git_access_protocol.blank? + true + else + protocol == current_application_settings.enabled_git_access_protocol + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c84c68f96f6..ffad5e17c78 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -13,7 +13,6 @@ module Gitlab "Cannot start with '-' or end in '.'." \ end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -22,7 +21,6 @@ module Gitlab "can contain only letters, digits, '_', '.', dash and space." end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -32,7 +30,6 @@ module Gitlab "It must start with letter, digit or '_'." end - def project_path_regex @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end @@ -42,7 +39,6 @@ module Gitlab "Cannot start with '-', end in '.git' or end in '.atom'" \ end - def file_name_regex @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end @@ -59,7 +55,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end - def directory_traversal_regex @directory_traversal_regex ||= /\.{2}/.freeze end @@ -68,7 +63,6 @@ module Gitlab "cannot include directory traversal. " end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 32c1c9ec5bb..67a5f368bdb 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash - def groups get_raw(Gitlab::Saml::Config.groups) end @@ -13,7 +12,6 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end - end end end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb index 0f40c00f547..574c3a4b28c 100644 --- a/lib/gitlab/saml/config.rb +++ b/lib/gitlab/saml/config.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class Config - class << self def options Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } @@ -15,7 +14,6 @@ module Gitlab options[:external_groups] end end - end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8943022612c..f253dc7477e 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -6,7 +6,6 @@ module Gitlab module Saml class User < Gitlab::OAuth::User - def save super('SAML') end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index ae85b294d31..104280f520a 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,18 +25,18 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill('SIGTERM', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + "#{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb new file mode 100644 index 00000000000..760ff3e614a --- /dev/null +++ b/lib/gitlab/template/base_template.rb @@ -0,0 +1,67 @@ +module Gitlab + module Template + class BaseTemplate + def initialize(path) + @path = path + end + + def name + File.basename(@path, self.class.extension) + end + + def content + File.read(@path) + end + + class << self + def all + self.categories.keys.flat_map { |cat| by_category(cat) } + end + + def find(key) + file_name = "#{key}#{self.extension}" + + directory = select_directory(file_name) + directory ? new(File.join(category_directory(directory), file_name)) : nil + end + + def categories + raise NotImplementedError + end + + def extension + raise NotImplementedError + end + + def base_dir + raise NotImplementedError + end + + def by_category(category) + templates_for_directory(category_directory(category)) + end + + def category_directory(category) + File.join(base_dir, categories[category]) + end + + private + + def select_directory(file_name) + categories.keys.find do |category| + File.exist?(File.join(category_directory(category), file_name)) + end + end + + def templates_for_directory(dir) + dir << '/' unless dir.end_with?('/') + Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) } + end + + def filter_regex + @filter_reges ||= /#{Regexp.escape(extension)}\z/ + end + end + end + end +end diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore.rb new file mode 100644 index 00000000000..964fbfd4de3 --- /dev/null +++ b/lib/gitlab/template/gitignore.rb @@ -0,0 +1,22 @@ +module Gitlab + module Template + class Gitignore < BaseTemplate + class << self + def extension + '.gitignore' + end + + def categories + { + "Languages" => '', + "Global" => 'Global' + } + end + + def base_dir + Rails.root.join('vendor/gitignore') + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml.rb new file mode 100644 index 00000000000..7f480fe33c0 --- /dev/null +++ b/lib/gitlab/template/gitlab_ci_yml.rb @@ -0,0 +1,27 @@ +module Gitlab + module Template + class GitlabCiYml < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + '.gitlab-ci.yml' + end + + def categories + { + "General" => '', + "Pages" => 'Pages' + } + end + + def base_dir + Rails.root.join('vendor/gitlab-ci-yml') + end + end + end + end +end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb new file mode 100644 index 00000000000..b290c716f97 --- /dev/null +++ b/lib/gitlab/timeless.rb @@ -0,0 +1,16 @@ +module Gitlab + module Timeless + def self.timeless(model, &block) + original_record_timestamps = model.record_timestamps + model.record_timestamps = false + + if block.arity.abs == 1 + block.call(model) + else + block.call + end + ensure + model.record_timestamps = original_record_timestamps + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 7d02fe3c971..19dad699edf 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -4,10 +4,20 @@ module Gitlab regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) content.gsub(regexp) { |url| new(url).masked_url } + rescue Addressable::URI::InvalidURIError + content.gsub(regexp, '') + end + + def self.valid?(url) + Addressable::URI.parse(url.strip) + + true + rescue Addressable::URI::InvalidURIError + false end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url) + @url = Addressable::URI.parse(url.strip) @credentials = credentials end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index d1b42c1f9b9..c0f85e9b3a8 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,7 +1,23 @@ module Gitlab - module UserAccess - def self.allowed?(user) - return false if user.blocked? + class UserAccess + attr_reader :user, :project + + def initialize(user, project: nil) + @user = user + @project = project + end + + def can_do_action?(action) + @permission_cache ||= {} + @permission_cache[action] ||= user.can?(action, project) + end + + def cannot_do_action?(action) + !can_do_action?(action) + end + + def allowed? + return false if user.blank? || user.blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -9,5 +25,31 @@ module Gitlab true end + + def can_push_to_branch?(ref) + return false unless user + + if project.protected_branch?(ref) && !project.developers_can_push_to_protected_branch?(ref) + user.can?(:push_code_to_protected_branches, project) + else + user.can?(:push_code, project) + end + end + + def can_merge_to_branch?(ref) + return false unless user + + if project.protected_branch?(ref) && !project.developers_can_merge_to_protected_branch?(ref) + user.can?(:push_code_to_protected_branches, project) + else + user.can?(:push_code, project) + end + end + + def can_read_project? + return false unless user + + user.can?(:read_project, project) + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 40e8299c36b..6aeb49c0219 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -38,12 +38,10 @@ module Gitlab end def send_git_diff(repository, diff_refs) - from, to = diff_refs - params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => from.sha, - 'ShaTo' => to.sha + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha } [ @@ -52,6 +50,31 @@ module Gitlab ] end + def send_git_patch(repository, diff_refs) + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha + } + + [ + SEND_DATA_HEADER, + "git-format-patch:#{encode(params)}" + ] + end + + def send_artifacts_entry(build, entry) + params = { + 'Archive' => build.artifacts_file.path, + 'Entry' => Base64.encode64(entry.path) + } + + [ + SEND_DATA_HEADER, + "artifacts-entry:#{encode(params)}" + ] + end + protected def encode(hash) diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 8c309efc7b8..f818dc78d34 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -1,175 +1,27 @@ -require 'cgi' - module Rouge module Formatters - class HTMLGitlab < Rouge::Formatter + class HTMLGitlab < Rouge::Formatters::HTML tag 'html_gitlab' # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # - # [+nowrap+] If set to True, don't wrap the output at all, not - # even inside a <tt><pre></tt> tag (default: false). - # [+cssclass+] CSS class for the wrapping <tt><div></tt> tag - # (default: 'highlight'). - # [+linenos+] If set to 'table', output line numbers as a table - # with two cells, one containing the line numbers, - # the other the whole code. This is copy paste friendly, - # but may cause alignment problems with some browsers - # or fonts. If set to 'inline', the line numbers will - # be integrated in the <tt><pre></tt> tag that contains - # the code (default: nil). # [+linenostart+] The line number for the first line (default: 1). - # [+lineanchors+] If set to true the formatter will wrap each output - # line in an anchor tag with a name of L-linenumber. - # This allows easy linking to certain lines - # (default: false). - # [+lineanchorsid+] If lineanchors is true the name of the anchors can - # be changed with lineanchorsid to e.g. foo-linenumber - # (default: 'L'). - # [+anchorlinenos+] If set to true, will wrap line numbers in <tt><a></tt> - # tags. Used in combination with linenos and lineanchors - # (default: false). - # [+inline_theme+] Inline CSS styles for the <pre> tag (default: false). - def initialize( - nowrap: false, - cssclass: 'highlight', - linenos: nil, - linenostart: 1, - lineanchors: false, - lineanchorsid: 'L', - anchorlinenos: false, - inline_theme: nil - ) - @nowrap = nowrap - @cssclass = cssclass - @linenos = linenos + def initialize(linenostart: 1) @linenostart = linenostart - @lineanchors = lineanchors - @lineanchorsid = lineanchorsid - @anchorlinenos = anchorlinenos - @inline_theme = Theme.find(inline_theme).new if inline_theme.is_a?(String) - end - - def render(tokens) - case @linenos - when 'table' - render_tableized(tokens) - when 'inline' - render_untableized(tokens) - else - render_untableized(tokens) - end - end - - alias_method :format, :render - - private - - def render_untableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap - html << wrap_lines(data[:code]) - html << "</code></pre>\n" unless @nowrap - html - end - - def render_tableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<div class=\"#{@cssclass}\">" unless @nowrap - html << '<table><tbody>' - html << "<td class=\"linenos\"><pre>" - html << wrap_linenos(data[:numbers]) - html << '</pre></td>' - html << "<td class=\"lines\"><pre><code>" - html << wrap_lines(data[:code]) - html << '</code></pre></td>' - html << '</tbody></table>' - html << '</div>' unless @nowrap - html - end - - def process_tokens(tokens) - rendered = [] - current_line = '' - - tokens.each do |tok, val| - # In the case of multi-line values (e.g. comments), we need to apply - # styling to each line since span elements are inline. - val.lines.each do |line| - stripped = line.chomp - current_line << span(tok, stripped) - - if line.end_with?("\n") - rendered << current_line - current_line = '' - end - end - end - - # Add leftover text - rendered << current_line if current_line.present? - - num_lines = rendered.size - numbers = (@linenostart..num_lines + @linenostart - 1).to_a - - { numbers: numbers, code: rendered } + @line_number = linenostart end - def wrap_linenos(numbers) - if @anchorlinenos - numbers.map! do |number| - "<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>" - end - end - numbers.join("\n") - end - - def wrap_lines(lines) - if @lineanchors - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - - if @linenos == 'inline' - "<a name=\"L#{number}\"></a>" \ - "<span class=\"linenos\">#{number}</span>" \ - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - else - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - end - end - lines.join("\n") - else - if @linenos == 'inline' - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - "<span class=\"linenos\">#{number}</span>#{line}" - end - lines.join("\n") - else - lines.join("\n") - end - end - end + def stream(tokens, &b) + is_first = true + token_lines(tokens) do |line| + yield "\n" unless is_first + is_first = false - def span(tok, val) - # http://stackoverflow.com/a/1600584/2587286 - val = CGI.escapeHTML(val) + yield %(<span id="LC#{@line_number}" class="line">) + line.each { |token, value| yield span(token, value) } + yield %(</span>) - if tok.shortname.empty? - val - else - if @inline_theme - rules = @inline_theme.style_for(tok).rendered_rules - "<span style=\"#{rules.to_a.join(';')}\"#{val}</span>" - else - "<span class=\"#{tok.shortname}\">#{val}</span>" - end + @line_number += 1 end end end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..4a4892a2e07 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -49,7 +49,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..0b93d7f292f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -93,7 +93,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 030ee8bafcb..993112aee3b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -4,7 +4,7 @@ namespace :gemojione do require 'digest/sha2' require 'json' - dir = Gemojione.index.images_path + dir = Gemojione.images_path digests = [] aliases = Hash.new { |hash, key| hash[key] = [] } aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') @@ -13,7 +13,7 @@ namespace :gemojione do aliases[real_name] << alias_name end - AwardEmoji.emojis.map do |name, emoji_hash| + Gitlab::AwardEmoji.emojis.map do |name, emoji_hash| fpath = File.join(dir, "#{emoji_hash['unicode']}.png") digest = Digest::SHA256.file(fpath).hexdigest @@ -50,9 +50,14 @@ namespace :gemojione do SIZE = 20 RETINA = SIZE * 2 + # Update these values to the width and height of the spritesheet when + # new emoji are added. + SPRITESHEET_WIDTH = 860 + SPRITESHEET_HEIGHT = 840 + Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + FileUtils.cp_r(Gemojione.images_path, tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| @@ -64,7 +69,7 @@ namespace :gemojione do # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" - SpriteFactory.run!(File.join(tmpdir, 'images'), { + SpriteFactory.run!(File.join(tmpdir, 'png'), { output_style: style_path, output_image: "app/assets/images/emoji.png", selector: '.emoji-', @@ -97,7 +102,7 @@ namespace :gemojione do only screen and (min-resolution: 192dpi), only screen and (min-resolution: 2dppx) { background-image: image-url('emoji@2x.png'); - background-size: 840px 820px; + background-size: #{SPRITESHEET_WIDTH}px #{SPRITESHEET_HEIGHT}px; } } CSS @@ -107,7 +112,7 @@ namespace :gemojione do # Now do it again but for Retina Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + FileUtils.cp_r(Gemojione.images_path, tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| @@ -116,7 +121,7 @@ namespace :gemojione do end # Combine the resized assets into a packed sprite and re-generate the SCSS - SpriteFactory.run!(File.join(tmpdir, 'images'), { + SpriteFactory.run!(File.join(tmpdir), { output_image: "app/assets/images/emoji@2x.png", style: false, nocomments: true, diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9ee72fde92f..b43ee5b3383 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,12 +33,13 @@ namespace :gitlab do unless backup.skipped?('db') unless ENV['force'] == 'yes' - warning = warning = <<-MSG.strip_heredoc + warning = <<-MSG.strip_heredoc Before restoring the database we recommend removing all existing tables to avoid future upgrade problems. Be aware that if you have custom tables in the GitLab database these tables and all data will be removed. MSG + puts warning.color(:red) ask_to_continue puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake new file mode 100644 index 00000000000..c2c6031db67 --- /dev/null +++ b/lib/tasks/gitlab/import_export.rake @@ -0,0 +1,13 @@ +namespace :gitlab do + namespace :import_export do + desc "GitLab | Show Import/Export version" + task version: :environment do + puts "Import/Export v#{Gitlab::ImportExport.version}" + end + + desc "GitLab | Display exported DB structure" + task data: :environment do + puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true) + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake new file mode 100644 index 00000000000..84aa2e8507a --- /dev/null +++ b/lib/tasks/gitlab/track_deployment.rake @@ -0,0 +1,9 @@ +namespace :gitlab do + desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' + task track_deployment: :environment do + metric = Gitlab::Metrics::Metric. + new('deployments', version: Gitlab::VERSION) + + Gitlab::Metrics.submit_metrics([metric.to_hash]) + end +end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake deleted file mode 100644 index 4fd48cccb1d..00000000000 --- a/lib/tasks/gitlab/update_gitignore.rake +++ /dev/null @@ -1,46 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update gitignore" - task :update_gitignore do - unless clone_gitignores - puts "Cloning the gitignores failed".color(:red) - return - end - - remove_unneeded_files(gitignore_directory) - remove_unneeded_files(global_directory) - - puts "Done".color(:green) - end - - def clone_gitignores - FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) - FileUtils.cd vendor_directory - - system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') - end - - # Retain only certain files: - # - The LICENSE, because we have to - # - The sub dir global - # - The gitignores themself - # - Dir.entires returns also the entries '.' and '..' - def remove_unneeded_files(path) - Dir.foreach(path) do |file| - FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ - end - end - - private - - def vendor_directory - Rails.root.join('vendor') - end - - def gitignore_directory - File.join(vendor_directory, 'gitignore') - end - - def global_directory - File.join(gitignore_directory, 'Global') - end -end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake new file mode 100644 index 00000000000..4f76dad7286 --- /dev/null +++ b/lib/tasks/gitlab/update_templates.rake @@ -0,0 +1,54 @@ +namespace :gitlab do + desc "GitLab | Update templates" + task :update_templates do + TEMPLATE_DATA.each { |template| update(template) } + end + + def update(template) + sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + dir = File.join(vendor_directory, sub_dir) + + unless clone_repository(template.repo_url, dir) + puts "Cloning the #{sub_dir} templates failed".red + return + end + + remove_unneeded_files(dir, template.cleanup_regex) + puts "Done".green + end + + def clone_repository(url, directory) + FileUtils.rm_rf(directory) if Dir.exist?(directory) + + system("git clone #{url} --depth=1 --branch=master #{directory}") + end + + # Retain only certain files: + # - The LICENSE, because we have to + # - The sub dirs so we can organise the file by category + # - The templates themself + # - Dir.entries returns also the entries '.' and '..' + def remove_unneeded_files(directory, regex) + Dir.foreach(directory) do |file| + FileUtils.rm_rf(File.join(directory, file)) unless file =~ regex + end + end + + private + + Template = Struct.new(:repo_url, :cleanup_regex) + TEMPLATE_DATA = [ + Template.new( + "https://github.com/github/gitignore.git", + /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ + ), + Template.new( + "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", + /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/ + ) + ] + + def vendor_directory + Rails.root.join('vendor') + end +end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index c5666d49e61..21c0e5f1d41 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -6,8 +6,6 @@ task :test do end unless Rails.env.production? - require 'coveralls/rake/task' - Coveralls::RakeTask.new desc "GitLab | Run all tests on CI with simplecov" - task :test_ci => [:rubocop, :brakeman, 'teaspoon', :spinach, :spec, 'coveralls:push'] + task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec] end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index d4291f012d3..41dee5fdc06 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -3,7 +3,6 @@ require "fileutils" # Taken from: Rack::Test::UploadedFile class UploadedFile - # The filename, *not* including the path, of the "uploaded" file attr_reader :original_filename |