diff options
Diffstat (limited to 'lib')
178 files changed, 4269 insertions, 1118 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..3d7d67510a8 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -26,38 +26,41 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers - mount ::API::Groups + mount ::API::AwardEmoji + mount ::API::Branches + mount ::API::Builds + mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::DeployKeys + mount ::API::Files mount ::API::GroupMembers - mount ::API::Users - mount ::API::Projects - mount ::API::Repositories + mount ::API::Groups + mount ::API::Internal mount ::API::Issues - mount ::API::Milestones - mount ::API::Session + mount ::API::Keys + mount ::API::Labels + mount ::API::LicenseTemplates mount ::API::MergeRequests + mount ::API::Milestones + mount ::API::Namespaces mount ::API::Notes - mount ::API::Internal - mount ::API::SystemHooks - mount ::API::ProjectSnippets - mount ::API::ProjectMembers - mount ::API::DeployKeys mount ::API::ProjectHooks + mount ::API::ProjectMembers + mount ::API::ProjectSnippets + mount ::API::Projects + mount ::API::Repositories + mount ::API::Runners mount ::API::Services - mount ::API::Files - mount ::API::Commits - mount ::API::CommitStatuses - mount ::API::Namespaces - mount ::API::Branches - mount ::API::Labels + mount ::API::Session mount ::API::Settings - mount ::API::Keys + mount ::API::SidekiqMetrics + mount ::API::Subscriptions + mount ::API::SystemHooks mount ::API::Tags + mount ::API::Templates + mount ::API::Todos mount ::API::Triggers - mount ::API::Builds + mount ::API::Users mount ::API::Variables - mount ::API::Runners - mount ::API::Licenses - mount ::API::Subscriptions - mount ::API::Gitignores end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb new file mode 100644 index 00000000000..c4fa1838b5a --- /dev/null +++ b/lib/api/award_emoji.rb @@ -0,0 +1,115 @@ +module API + class AwardEmoji < Grape::API + before { authenticate! } + AWARDABLES = [Issue, MergeRequest] + + resource :projects do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.to_s.underscore.pluralize + awardable_id_string = "#{awardable_type.to_s.underscore}_id" + + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get endpoint do + if can_read_awardable? + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or mr + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:awardable_id/award_emoji + post endpoint do + required_attributes! [:name] + + not_found!('Award Emoji') unless can_read_awardable? + + award = awardable.award_emoji.new(name: params[:name], user: current_user) + + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + end + + helpers do + def can_read_awardable? + ability = "read_#{awardable.class.to_s.underscore}".to_sym + + can?(current_user, ability, awardable) + end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + noteable.notes.find(params[:note_id]) + else + noteable + end + end + end + + def noteable + if params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + else + user_project.merge_requests.find(params[:merge_request_id]) + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 231840148d9..d467eb9d474 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -25,7 +25,7 @@ module API # branch (required) - The name of the branch # Example Request: # GET /projects/:id/repository/branches/:branch - get ':id/repository/branches/:branch', requirements: { branch: /.*/ } do + get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } not_found!("Branch") unless @branch present @branch, with: Entities::RepoObject, project: user_project @@ -39,8 +39,7 @@ module API # Example Request: # PUT /projects/:id/repository/branches/:branch/protect put ':id/repository/branches/:branch/protect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) @@ -59,8 +58,7 @@ module API # Example Request: # PUT /projects/:id/repository/branches/:branch/unprotect put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) @@ -101,7 +99,7 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.*/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). execute(params[:branch]) diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 645e2dda0b7..d36047acd1f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -13,7 +13,6 @@ module API # Example Request: # GET /projects/:id/builds get ':id/builds' do - builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -33,10 +32,10 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.pipelines.find_by_sha(params[:sha]) - return not_found! unless commit + return not_found! unless user_project.commit(params[:sha]) - builds = commit.builds.order('id DESC') + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build, @@ -142,7 +141,7 @@ module API return not_found!(build) unless build return forbidden!('Build is not retryable') unless build.retryable? - build = Ci::Build.retry(build) + build = Ci::Build.retry(build, current_user) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cc29c7ef428..8e03c08f47b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -54,10 +54,19 @@ module API class BasicProjectDetails < Grape::Entity expose :id + expose :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace end + class SharedGroup < Grape::Entity + expose :group_id + expose :group_name do |group_link, options| + group_link.group.name + end + expose :group_access, as: :group_access_level + end + class Project < Grape::Entity expose :id, :description, :default_branch, :tag_list expose :public?, as: :public @@ -77,6 +86,9 @@ module API expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds + expose :shared_with_groups do |project, options| + SharedGroup.represent(project.project_group_links.all, options) + end end class ProjectMember < UserBasic @@ -93,6 +105,7 @@ module API class GroupDetail < Group expose :projects, using: Entities::Project + expose :shared_projects, using: Entities::Project end class GroupMember < UserBasic @@ -187,7 +200,6 @@ module API expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels - expose :description expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds @@ -196,6 +208,8 @@ module API merge_request.subscribed?(options[:current_user]) end expose :user_notes_count + expose :should_remove_source_branch?, as: :should_remove_source_branch + expose :force_remove_source_branch?, as: :force_remove_source_branch end class MergeRequestChanges < MergeRequest @@ -225,6 +239,14 @@ module API expose(:downvote?) { |note| false } end + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + class MRNote < Grape::Entity expose :note expose :author, using: Entities::UserBasic @@ -232,9 +254,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } - expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } - expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } + expose(:path) { |note| note.diff_file.try(:file_path) if note.diff_note? } + expose(:line) { |note| note.diff_line.try(:new_line) if note.diff_note? } + expose(:line_type) { |note| note.diff_line.try(:type) if note.diff_note? } expose :author, using: Entities::UserBasic expose :created_at end @@ -264,6 +286,31 @@ module API expose :id, :project_id, :group_id, :group_access end + class Todo < Grape::Entity + expose :id + expose :project, using: Entities::BasicProjectDetails + expose :author, using: Entities::UserBasic + expose :action_name + expose :target_type + + expose :target do |todo, options| + Entities.const_get(todo.target_type).represent(todo.target, options) + end + + expose :target_url do |todo, options| + target_type = todo.target_type.underscore + target_url = "namespace_project_#{target_type}_url" + target_anchor = "note_#{todo.note_id}" if todo.note_id? + + Gitlab::Application.routes.url_helpers.public_send(target_url, + todo.project.namespace, todo.project, todo.target, anchor: target_anchor) + end + + expose :body + expose :state + expose :created_at + end + class Namespace < Grape::Entity expose :id, :path, :kind end @@ -368,6 +415,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity @@ -415,6 +463,7 @@ module API class RunnerDetails < Runner expose :tag_list expose :run_untagged + expose :locked expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -436,11 +485,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: User expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: RepoCommit do |repo_obj, _options| - if repo_obj.respond_to?(:commit) - repo_obj.commit.commit_data - end - end + expose :commit, with: RepoCommit expose :runner, with: Runner end @@ -464,11 +509,11 @@ module API expose :content end - class GitignoresList < Grape::Entity + class TemplatesList < Grape::Entity expose :name end - class Gitignore < Grape::Entity + class Template < Grape::Entity expose :name, :content end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb deleted file mode 100644 index 270c9501dd2..00000000000 --- a/lib/api/gitignores.rb +++ /dev/null @@ -1,29 +0,0 @@ -module API - class Gitignores < Grape::API - - # Get the list of the available gitignore templates - # - # Example Request: - # GET /gitignores - get 'gitignores' do - present Gitlab::Gitignore.all, with: Entities::GitignoresList - end - - # Get the text for a specific gitignore - # - # Parameters: - # name (required) - The name of a license - # - # Example Request: - # GET /gitignores/Elixir - # - get 'gitignores/:name' do - required_attributes! [:name] - - gitignore = Gitlab::Gitignore.find(params[:name]) - not_found!('.gitignore') unless gitignore - - present gitignore, with: Entities::Gitignore - end - end -end diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb index ab9b7c602b5..dbe5bb08d3f 100644 --- a/lib/api/group_members.rb +++ b/lib/api/group_members.rb @@ -77,7 +77,7 @@ module API member = group.group_members.find_by(user_id: params[:user_id]) if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}",404) + render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) else member.destroy end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index de5959e3aae..77e407b54c5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -9,9 +9,13 @@ module API [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end + def find_user_by_private_token + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + end + def current_user - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + @current_user ||= (find_user_by_private_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil @@ -33,7 +37,7 @@ module API identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers - if !!(identifier =~ /^[0-9]+$/) + if !!(identifier =~ /\A[0-9]+\z/) identifier.to_i else identifier diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 3ac7b50c4ce..d5dfba5e0cc 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -13,6 +13,7 @@ module API # action - git action (git-upload-pack or git-receive-pack) # ref - branch name # forced_push - forced_push + # protocol - Git access protocol being used, e.g. HTTP or SSH # helpers do @@ -20,11 +21,23 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do - Gitlab::Metrics.action = 'Grape#/internal/allowed' - status 200 actor = @@ -34,24 +47,26 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) + protocol = params[:protocol] access = if wiki? - Gitlab::GitAccessWiki.new(actor, project) + Gitlab::GitAccessWiki.new(actor, project, protocol) else - Gitlab::GitAccess.new(actor, project) + Gitlab::GitAccess.new(actor, project, protocol) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = project.repository.path_to_repo + end + + response end # diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4c43257c48a..8a03a41e9c5 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -59,6 +59,41 @@ module API end end + resource :groups do + # Get a list of group issues + # + # Parameters: + # id (required) - The ID of a group + # state (optional) - Return "opened" or "closed" issues + # labels (optional) - Comma-separated list of label names + # milestone (optional) - Milestone title + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` + # + # Example Requests: + # GET /groups/:id/issues + # GET /groups/:id/issues?state=opened + # GET /groups/:id/issues?state=closed + # GET /groups/:id/issues?labels=foo + # GET /groups/:id/issues?labels=foo,bar + # GET /groups/:id/issues?labels=foo,bar&state=opened + # GET /groups/:id/issues?milestone=1.0.0 + # GET /groups/:id/issues?milestone=1.0.0&state=closed + get ":id/issues" do + group = find_group(params[:id]) + + params[:state] ||= 'opened' + params[:group_id] = group.id + params[:milestone_title] = params.delete(:milestone) + params[:label_name] = params.delete(:labels) + params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + issues = IssuesFinder.new(current_user, params).execute + + present paginate(issues), with: Entities::Issue, current_user: current_user + end + end + resource :projects do # Get a list of project issues # diff --git a/lib/api/licenses.rb b/lib/api/license_templates.rb index be0e113fbcb..d0552299ed0 100644 --- a/lib/api/licenses.rb +++ b/lib/api/license_templates.rb @@ -1,6 +1,6 @@ module API - # Licenses API - class Licenses < Grape::API + # License Templates API + class LicenseTemplates < Grape::API PROJECT_TEMPLATE_REGEX = /[\<\{\[] (project|description| diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 0e94efd4acd..4fcdf8968c9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -233,8 +233,8 @@ module API render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? - if params[:sha] && merge_request.source_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.source_sha}", 409) + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) end merge_params = { diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 132043cf3f7..7a0cb7c99f3 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -115,7 +115,6 @@ module API issues = IssuesFinder.new(current_user, finder_params).execute present paginate(issues), with: Entities::Issue, current_user: current_user end - end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d4fcfd3d4d3..8bfa998dc53 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -144,7 +144,7 @@ module API helpers do def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore.downcase}".to_sym + "read_#{noteable.class.to_s.underscore}".to_sym end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index ccca65cbe1c..6bb70bc8bc3 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -28,7 +28,6 @@ module API present @hook, with: Entities::ProjectHook end - # Add hook to project # # Parameters: diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index b703da0557a..6a0b3e7d134 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -4,7 +4,6 @@ module API before { authenticate! } resource :projects do - # Get a project team members # # Parameters: diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5a22d14988f..6d2a6f3946c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,7 +25,11 @@ module API @projects = current_user.authorized_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + if params[:simple] + present @projects, with: Entities::BasicProjectDetails, user: current_user + else + present @projects, with: Entities::ProjectWithAccess, user: current_user + end end # Get an owned projects list for authenticated user @@ -341,7 +345,6 @@ module API else not_found!("Source Project") end - end # Remove a forked_from relationship @@ -418,7 +421,6 @@ module API present paginate(projects), with: Entities::Project end - # Get a users list # # Example Request: diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 4faba9dc87b..ecc8f2fc5a2 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] + attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else @@ -96,9 +96,14 @@ module API runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) - Ci::RunnerProject.create(runner: runner, project: user_project) - present runner, with: Entities::Runner + runner_project = runner.assign_to(user_project) + + if runner_project.persisted? + present runner, with: Entities::Runner + else + conflict!("Runner was already enabled for this project") + end end # Disable project's runner @@ -163,6 +168,7 @@ module API def authenticate_enable_runner!(runner) forbidden!("Runner is shared") if runner.is_shared? + forbidden!("Runner is locked") if runner.locked? return if current_user.is_admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 203f04a6259..fc8598daa32 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -4,7 +4,6 @@ module API before { authenticate! } before { authorize_admin_project } - resource :projects do # Set <service_slug> service for project # diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 3e1ed3fe5c7..7b675e05fbb 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -61,7 +61,7 @@ module API # tag_name (required) - The name of the tag # Example Request: # DELETE /projects/:id/repository/tags/:tag - delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -83,7 +83,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # POST /projects/:id/repository/tags/:tag_name/release - post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = CreateReleaseService.new(user_project, current_user). @@ -104,7 +104,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # PUT /projects/:id/repository/tags/:tag_name/release - put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = UpdateReleaseService.new(user_project, current_user). diff --git a/lib/api/templates.rb b/lib/api/templates.rb new file mode 100644 index 00000000000..18408797756 --- /dev/null +++ b/lib/api/templates.rb @@ -0,0 +1,36 @@ +module API + class Templates < Grape::API + TEMPLATE_TYPES = { + gitignores: Gitlab::Template::Gitignore, + gitlab_ci_ymls: Gitlab::Template::GitlabCiYml + }.freeze + + TEMPLATE_TYPES.each do |template, klass| + # Get the list of the available template + # + # Example Request: + # GET /gitignores + # GET /gitlab_ci_ymls + get template.to_s do + present klass.all, with: Entities::TemplatesList + end + + # Get the text for a specific template + # + # Parameters: + # name (required) - The name of a template + # + # Example Request: + # GET /gitignores/Elixir + # GET /gitlab_ci_ymls/Ruby + get "#{template}/:name" do + required_attributes! [:name] + + new_template = klass.find(params[:name]) + not_found!(template.to_s.singularize) unless new_template + + present new_template, with: Entities::Template + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb new file mode 100644 index 00000000000..2a6bfa98ca4 --- /dev/null +++ b/lib/api/todos.rb @@ -0,0 +1,82 @@ +module API + # Todos API + class Todos < Grape::API + before { authenticate! } + + ISSUABLE_TYPES = { + 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'issues' => ->(id) { find_project_issue(id) } + } + + resource :projects do + ISSUABLE_TYPES.each do |type, finder| + type_id_str = "#{type.singularize}_id".to_sym + + # Create a todo on an issuable + # + # Parameters: + # id (required) - The ID of a project + # issuable_id (required) - The ID of an issuable + # Example Request: + # POST /projects/:id/issues/:issuable_id/todo + # POST /projects/:id/merge_requests/:issuable_id/todo + post ":id/#{type}/:#{type_id_str}/todo" do + issuable = instance_exec(params[type_id_str], &finder) + todo = TodoService.new.mark_todo(issuable, current_user).first + + if todo + present todo, with: Entities::Todo, current_user: current_user + else + not_modified! + end + end + end + end + + resource :todos do + helpers do + def find_todos + TodosFinder.new(current_user, params).execute + end + end + + # Get a todo list + # + # Example Request: + # GET /todos + # + get do + todos = find_todos + + present paginate(todos), with: Entities::Todo, current_user: current_user + end + + # Mark a todo as done + # + # Parameters: + # id: (required) - The ID of the todo being marked as done + # + # Example Request: + # DELETE /todos/:id + # + delete ':id' do + todo = current_user.todos.find(params[:id]) + todo.done + + present todo, with: Entities::Todo, current_user: current_user + end + + # Mark all todos as done + # + # Example Request: + # DELETE /todos + # + delete do + todos = find_todos + todos.each(&:done) + + present paginate(Kaminari.paginate_array(todos)), with: Entities::Todo, current_user: current_user + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/banzai.rb b/lib/banzai.rb index b467413a7dd..9ebe379f454 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,12 +3,12 @@ module Banzai Renderer.render(text, context) end - def self.render_result(text, context = {}) - Renderer.render_result(text, context) + def self.cache_collection_render(texts_and_contexts) + Renderer.cache_collection_render(texts_and_contexts) end - def self.pre_process(text, context) - Renderer.pre_process(text, context) + def self.render_result(text, context = {}) + Renderer.render_result(text, context) end def self.post_process(html, context) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index db95d7c908b..d77a5e3ff09 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -103,7 +103,7 @@ module Banzai ref_pattern = object_class.reference_pattern link_pattern = object_class.link_reference_pattern - each_node do |node| + nodes.each do |node| if text_node?(node) && ref_pattern replace_text_when_pattern_matches(node, ref_pattern) do |content| object_link_filter(content, ref_pattern) @@ -160,11 +160,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attribute( - original: link_text || match, - project: project.id, - object_sym => object.id - ) + data = data_attributes_for(link_text || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -183,6 +179,14 @@ module Banzai end end + def data_attributes_for(text, project, object) + data_attribute( + original: text, + project: project.id, + object_sym => object.id + ) + end + def object_link_text_extras(object, matches) extras = [] @@ -206,6 +210,56 @@ module Banzai text end + # Returns a Hash containing all object references (e.g. issue IDs) per the + # project they belong to. + def references_per_project + @references_per_project ||= begin + refs = Hash.new { |hash, key| hash[key] = Set.new } + + regex = Regexp.union(object_class.reference_pattern, + object_class.link_reference_pattern) + + nodes.each do |node| + node.to_html.scan(regex) do + project = $~[:project] || current_project_path + symbol = $~[object_sym] + + refs[project] << symbol if object_class.reference_valid?(symbol) + end + end + + refs + end + end + + # Returns a Hash containing referenced projects grouped per their full + # path. + def projects_per_reference + @projects_per_reference ||= begin + hash = {} + refs = Set.new + + references_per_project.each do |project_ref, _| + refs << project_ref + end + + find_projects_for_paths(refs.to_a).each do |project| + hash[project.path_with_namespace] = project + end + + hash + end + end + + # Returns the projects for the given paths. + def find_projects_for_paths(paths) + Project.where_paths_in(paths).includes(:namespace) + end + + def current_project_path + @current_project_path ||= project.path_with_namespace + end + private def project_refs_cache diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb new file mode 100644 index 00000000000..d2c4b1e4d76 --- /dev/null +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -0,0 +1,71 @@ +module Banzai + module Filter + class BlockquoteFenceFilter < HTML::Pipeline::TextFilter + REGEX = %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?: + # Blockquote: + # >>> + # Anything, including code and HTML blocks + # >>> + + ^>>>\n + (?<quote> + (?: + # Any character that doesn't introduce a code or HTML block + (?! + ^``` + | + ^<[^>]+?>\n + ) + . + | + # A code block + \g<code> + | + # An HTML block + \g<html> + )+? + ) + \n>>>$ + ) + }mx.freeze + + def initialize(text, context = nil, result = nil) + super text, context, result + @text = @text.delete("\r") + end + + def call + @text.gsub(REGEX) do + if $~[:quote] + $~[:quote].gsub(/^/, "> ").gsub(/^> $/, ">") + else + $~[0] + end + end + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d25de900674..ae7d31cf191 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -61,7 +61,7 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end def emoji_pattern @@ -69,7 +69,7 @@ module Banzai end def emoji_filename(name) - "#{Emoji.emoji_filename(name)}.png" + "#{Gitlab::Emoji.emoji_filename(name)}.png" end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index f73ecfc9418..0a29c547a4d 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -3,17 +3,8 @@ module Banzai # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - doc.search('a').each do |node| - link = node.attr('href') - - next unless link - - # Skip non-HTTP(S) links - next unless link.start_with?('http') - - # Skip internal links - next if link.start_with?(internal_url) - + # Skip non-HTTP(S) links and internal links + doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| node.set_attribute('rel', 'nofollow noreferrer') node.set_attribute('target', '_blank') end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index ccd106860bd..f0fb6084a35 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -8,6 +8,10 @@ module Banzai # of the anchor, and then replace the img with the link-wrapped version. def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + div = doc.document.create_element( + 'div', + class: 'image-container' + ) link = doc.document.create_element( 'a', @@ -17,7 +21,10 @@ module Banzai ) link.children = img.clone - img.replace(link) + + div.children = link + + img.replace(div) end doc diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2496e704002..4042e9a4c25 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -11,13 +11,64 @@ module Banzai Issue end - def find_object(project, id) - project.get_issue(id) + def find_object(project, iid) + issues_per_project[project][iid] end def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) end + + def project_from_ref(ref) + projects_per_reference[ref || current_project_path] + end + + # Returns a Hash containing the issues per Project instance. + def issues_per_project + @issues_per_project ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + projects_per_reference.each do |path, project| + issue_ids = references_per_project[path] + + if project.default_issues_tracker? + issues = project.issues.where(iid: issue_ids.to_a) + else + issues = issue_ids.map { |id| ExternalIssue.new(id, project) } + end + + issues.each do |issue| + hash[project][issue.iid.to_i] = issue + end + end + + hash + end + end + + def object_link_title(object) + if object.is_a?(ExternalIssue) + "Issue in #{object.project.external_issue_tracker.title}" + else + super + end + end + + def data_attributes_for(text, project, object) + if object.is_a?(ExternalIssue) + data_attribute( + project: project.id, + external_issue: object.id, + reference_type: ExternalIssueReferenceFilter.reference_type + ) + else + super + end + end + + def find_projects_for_paths(paths) + super(paths).includes(:gitlab_issue_tracker_service) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e4d3f87d0aa..e258dc8e2bf 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -13,13 +13,13 @@ module Banzai end def self.references_in(text, pattern = Label.reference_pattern) - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ end end def references_in(text, pattern = Label.reference_pattern) - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| label = find_label($~[:project], $~[:label_id], $~[:label_name]) if label @@ -66,6 +66,10 @@ module Banzai LabelsHelper.render_colored_cross_project_label(object) end end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c753a84a20d..c59a80dd1c7 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,40 +7,13 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - nodes = Querying.css(doc, 'a.gfm[data-reference-type]') - visible = nodes_visible_to_user(nodes) - - nodes.each do |node| - unless visible.include?(node) - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) - end - end + Redactor.new(project, current_user).redact([doc]) doc end private - def nodes_visible_to_user(nodes) - per_type = Hash.new { |h, k| h[k] = [] } - visible = Set.new - - nodes.each do |node| - per_type[node.attr('data-reference-type')] << node - end - - per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, current_user) - - visible.merge(parser.nodes_visible_to_user(current_user, nodes)) - end - - visible - end - def current_user context[:current_user] end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d6f34c9cd8..bf058241cda 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -29,7 +29,7 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] = self.class.reference_type + attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index ea21c7b041c..c78da404607 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -14,6 +14,8 @@ module Banzai def call return doc unless linkable_files? + @uri_types = {} + doc.search('a:not(.gfm)').each do |el| process_link_attr el.attribute('href') end @@ -48,7 +50,7 @@ module Banzai uri.path = [ relative_url_root, context[:project].path_with_namespace, - path_type(file_path), + uri_type(file_path), ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') @@ -87,7 +89,7 @@ module Banzai return path unless request_path parts = request_path.split('/') - parts.pop if path_type(request_path) != 'tree' + parts.pop if uri_type(request_path) != :tree while path.start_with?('../') parts.pop @@ -98,45 +100,20 @@ module Banzai end def file_exists?(path) - return false if path.nil? - repository.blob_at(current_sha, path).present? || - repository.tree(current_sha, path).entries.any? - end - - # Get the type of the given path - # - # path - String path to check - # - # Examples: - # - # path_type('doc/README.md') # => 'blob' - # path_type('doc/logo.png') # => 'raw' - # path_type('doc/api') # => 'tree' - # - # Returns a String - def path_type(path) - unescaped_path = Addressable::URI.unescape(path) - - if tree?(unescaped_path) - 'tree' - elsif image?(unescaped_path) - 'raw' - else - 'blob' - end + path.present? && !!uri_type(path) end - def tree?(path) - repository.tree(current_sha, path).entries.any? - end + def uri_type(path) + @uri_types[path] ||= begin + unescaped_path = Addressable::URI.unescape(path) - def image?(path) - repository.blob_at(current_sha, path).try(:image?) + current_commit.uri_type(unescaped_path) + end end - def current_sha - context[:commit].try(:id) || - ref ? repository.commit(ref).try(:sha) : repository.head_commit.sha + def current_commit + @current_commit ||= context[:commit] || + ref ? repository.commit(ref) : repository.head_commit end def relative_url_root @@ -148,7 +125,7 @@ module Banzai end def repository - context[:project].try(:repository) + @repository ||= context[:project].try(:repository) end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 62a79c62e20..536b478979f 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -27,12 +27,17 @@ module Banzai highlighted = "<pre>#{code}</pre>" end - # Replace the parent `pre` element with the entire highlighted block - node.parent.replace(highlighted) + # Extracted to a method to measure it + replace_parent_pre_element(node, highlighted) end private + def replace_parent_pre_element(node, highlighted) + # Replace the parent `pre` element with the entire highlighted block + node.parent.replace(highlighted) + end + # Override Rouge::Plugins::Redcarpet#rouge_formatter def rouge_formatter(lexer) Rouge::Formatters::HTMLGitlab.new( diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index c0f503c9af3..45bb66dc99f 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -10,11 +10,11 @@ module Banzai def call return doc unless project - doc.search('a').each do |el| + doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') end - doc.search('img').each do |el| + doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el| process_link_attr el.attribute('src') end @@ -24,12 +24,7 @@ module Banzai protected def process_link_attr(html_attr) - return if html_attr.blank? - - uri = html_attr.value - if uri.starts_with?("/uploads/") - html_attr.value = build_url(uri).to_s - end + html_attr.value = build_url(html_attr.value).to_s end def build_url(uri) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 37a2779d453..269d5bf74fa 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,7 +8,6 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter - def call return doc unless project_wiki? @@ -29,7 +28,7 @@ module Banzai return if html_attr.blank? html_attr.value = apply_rewrite_rules(html_attr.value) - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb new file mode 100644 index 00000000000..bab6a9934d1 --- /dev/null +++ b/lib/banzai/note_renderer.rb @@ -0,0 +1,22 @@ +module Banzai + module NoteRenderer + # Renders a collection of Note instances. + # + # notes - The notes to render. + # project - The project to use for rendering/redacting. + # user - The user viewing the notes. + # path - The request path. + # wiki - The project's wiki. + # git_ref - The current Git reference. + def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil) + renderer = ObjectRenderer.new(project, + user, + requested_path: path, + project_wiki: wiki, + ref: git_ref, + pipeline: :note) + + renderer.render(notes, :note) + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb new file mode 100644 index 00000000000..9aef807c152 --- /dev/null +++ b/lib/banzai/object_renderer.rb @@ -0,0 +1,86 @@ +module Banzai + # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # + # Rendered Markdown is stored in an attribute in every object based on the + # name of the attribute containing the Markdown. For example, when the + # attribute `note` is rendered the HTML is stored in `note_html`. + class ObjectRenderer + attr_reader :project, :user + + # Make sure to set the appropriate pipeline in the `raw_context` attribute + # (e.g. `:note` for Note instances). + # + # project - A Project to use for rendering and redacting Markdown. + # user - The user viewing the Markdown/HTML documents, if any. + # context - A Hash containing extra attributes to use in the rendering + # pipeline. + def initialize(project, user = nil, raw_context = {}) + @project = project + @user = user + @raw_context = raw_context + end + + # Renders and redacts an Array of objects. + # + # objects - The objects to render + # attribute - The attribute containing the raw Markdown to render. + # + # Returns the same input objects. + def render(objects, attribute) + documents = render_objects(objects, attribute) + redacted = redact_documents(documents) + + objects.each_with_index do |object, index| + redacted_data = redacted[index] + object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.user_visible_reference_count = redacted_data[:visible_reference_count] + end + end + + # Renders the attribute of every given object. + def render_objects(objects, attribute) + render_attributes(objects, attribute) + end + + # Redacts the list of documents. + # + # Returns an Array containing the redacted documents. + def redact_documents(documents) + redactor = Redactor.new(project, user) + + redactor.redact(documents) + end + + # Returns a Banzai context for the given object and attribute. + def context_for(object, attribute) + context = base_context.merge(cache_key: [object, attribute]) + + if object.respond_to?(:author) + context[:author] = object.author + end + + context + end + + # Renders the attributes of a set of objects. + # + # Returns an Array of `Nokogiri::HTML::Document`. + def render_attributes(objects, attribute) + strings_and_contexts = objects.map do |object| + context = context_for(object, attribute) + + string = object.__send__(attribute) + + { text: string, context: context } + end + + Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| + Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + end + end + + def base_context + @base_context ||= @raw_context.merge(current_user: user, project: project) + end + end +end diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index d47ddfda4be..3c974f73176 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,7 +1,6 @@ module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) - end end end diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb index 50dc978b452..6cf219661d3 100644 --- a/lib/banzai/pipeline/pre_process_pipeline.rb +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -3,7 +3,8 @@ module Banzai class PreProcessPipeline < BasePipeline def self.filters FilterArray[ - Filter::YamlFrontMatterFilter + Filter::YamlFrontMatterFilter, + Filter::BlockquoteFenceFilter, ] end diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb new file mode 100644 index 00000000000..270990e7ab4 --- /dev/null +++ b/lib/banzai/pipeline/relative_link_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class RelativeLinkPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::RelativeLinkFilter + ] + end + end + end +end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb new file mode 100644 index 00000000000..0df3a72d1c4 --- /dev/null +++ b/lib/banzai/redactor.rb @@ -0,0 +1,82 @@ +module Banzai + # Class for removing Markdown references a certain user is not allowed to + # view. + class Redactor + attr_reader :user, :project + + # project - A Project to use for redacting links. + # user - The currently logged in user (if any). + def initialize(project, user = nil) + @project = project + @user = user + end + + # Redacts the references in the given Array of documents. + # + # This method modifies the given documents in-place. + # + # documents - A list of HTML documents containing references to redact. + # + # Returns the documents passed as the first argument. + def redact(documents) + all_document_nodes = document_nodes(documents) + + redact_document_nodes(all_document_nodes) + end + + # Redacts the given node documents + # + # data - An Array of a Hashes mapping an HTML document to nodes to redact. + def redact_document_nodes(all_document_nodes) + all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + visible = nodes_visible_to_user(all_nodes) + metadata = [] + + all_document_nodes.each do |entry| + nodes_for_document = entry[:nodes] + doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + metadata << doc_data + + nodes_for_document.each do |node| + next if visible.include?(node) + + doc_data[:visible_reference_count] -= 1 + # The reference should be replaced by the original text, + # which is not always the same as the rendered text. + text = node.attr('data-original') || node.text + node.replace(text) + end + end + + metadata + end + + # Returns the nodes visible to the current user. + # + # nodes - The input nodes to check. + # + # Returns a new Array containing the visible nodes. + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, user) + + visible.merge(parser.nodes_visible_to_user(user, nodes)) + end + + visible + end + + def document_nodes(documents) + documents.map do |document| + { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') } + end + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 3d7b9c4a024..6cf218aaa0d 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -133,8 +133,9 @@ module Banzai return {} if nodes.empty? ids = unique_attribute_values(nodes, attribute) + rows = collection_objects_for_ids(collection, ids) - collection.where(id: ids).each_with_object({}) do |row, hash| + rows.each_with_object({}) do |row, hash| hash[row.id] = row end end @@ -153,6 +154,31 @@ module Banzai values.to_a end + # Queries the collection for the objects with the given IDs. + # + # If the RequestStore module is enabled this method will only query any + # objects that have not yet been queried. For objects that have already + # been queried the object is returned from the cache. + def collection_objects_for_ids(collection, ids) + if RequestStore.active? + cache = collection_cache[collection_cache_key(collection)] + to_query = ids.map(&:to_i) - cache.keys + + unless to_query.empty? + collection.where(id: to_query).each { |row| cache[row.id] = row } + end + + cache.values + else + collection.where(id: ids) + end + end + + # Returns the cache key to use for a collection. + def collection_cache_key(collection) + collection.respond_to?(:model) ? collection.model : collection + end + # Processes the list of HTML documents and returns an Array containing all # the references. def process(documents) @@ -189,7 +215,7 @@ module Banzai end def find_projects_for_hash_keys(hash) - Project.where(id: hash.keys) + collection_objects_for_ids(Project, hash.keys) end private @@ -199,6 +225,12 @@ module Banzai def lazy(&block) Gitlab::Lazy.new(&block) end + + def collection_cache + RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| + hash[key] = {} + end + end end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 24076e3d9ec..f306079d833 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -25,7 +25,21 @@ module Banzai def issues_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, - Issue.all.includes(:author, :assignee, :project), + Issue.all.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members + ] + } + ), self.class.data_attribute ) end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index a12b0d19560..863f5725d3b 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -73,7 +73,7 @@ module Banzai def find_users(ids) return [] if ids.empty? - User.where(id: ids).to_a + collection_objects_for_ids(User, ids) end def find_users_for_groups(ids) @@ -85,7 +85,8 @@ module Banzai def find_users_for_projects(ids) return [] if ids.empty? - Project.where(id: ids).flat_map { |p| p.team.members.to_a } + collection_objects_for_ids(Project, ids). + flat_map { |p| p.team.members.to_a } end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c14a9c4c722..910687a7b6a 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -10,7 +10,7 @@ module Banzai # requiring XHTML, such as Atom feeds, need to call `post_process` on the # result, providing the appropriate `pipeline` option. # - # markdown - Markdown String + # text - Markdown String # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String @@ -29,14 +29,62 @@ module Banzai end end - def self.render_result(text, context = {}) - Pipeline[context[:pipeline]].call(text, context) + # Perform multiple render from an Array of Markdown String into an + # Array of HTML-safe String of HTML. + # + # As the rendered Markdown String can be already cached read all the data + # from the cache using Rails.cache.read_multi operation. If the Markdown String + # is not in the cache or it's not cacheable (no cache_key entry is provided in + # the context) the Markdown String is rendered and stored in the cache so the + # next render call gets the rendered HTML-safe String from the cache. + # + # For further explanation see #render method comments. + # + # texts_and_contexts - An Array of Hashes that contains the Markdown String (:text) + # an options passed to our HTML Pipeline (:context) + # + # If on the :context you specify a :cache_key entry will be used to retrieve it + # and cache the result of rendering the Markdown String. + # + # Returns an Array containing HTML-safe String instances. + # + # Example: + # texts_and_contexts + # => [{ text: '### Hello', + # context: { cache_key: [note, :note] } }] + def self.cache_collection_render(texts_and_contexts) + items_collection = texts_and_contexts.each_with_index do |item, index| + context = item[:context] + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + + item[:cache_key] = cache_key if cache_key + end + + cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + + items_in_cache = [] + items_not_in_cache = [] + + unless cacheable_items.empty? + items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) + items_not_in_cache = cacheable_items.reject do |item| + item[:rendered] = items_in_cache[item[:cache_key]] + items_in_cache.key?(item[:cache_key]) + end + end + + (items_not_in_cache + non_cacheable_items).each do |item| + item[:rendered] = render(item[:text], item[:context]) + Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] + end + + items_collection.map { |item| item[:rendered] } end - def self.pre_process(text, context) - pipeline = Pipeline[:pre_process] + def self.render_result(text, context = {}) + text = Pipeline[:pre_process].to_html(text, context) if text - pipeline.to_html(text, context) + Pipeline[context[:pipeline]].call(text, context) end # Perform post-processing on an HTML String @@ -82,5 +130,13 @@ module Banzai return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end + + # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. + # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key + # method. + def self.full_cache_multi_key(cache_key, pipeline_name) + return unless cache_key + Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + end end end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 9f270f7b387..260ac81f5fa 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -195,8 +195,7 @@ module Ci not_found! unless build authenticate_build_token!(build) - build.remove_artifacts_file! - build.remove_artifacts_metadata! + build.erase_artifacts! end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 0c41f22c7c5..bcc82969eb3 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,12 +28,9 @@ module Ci post "register" do required_attributes! [:token] - attributes = { description: params[:description], - tag_list: params[:tag_list] } - - unless params[:run_untagged].nil? - attributes[:run_untagged] = params[:run_untagged] - end + attributes = attributes_for_keys( + [:description, :tag_list, :run_untagged, :locked] + ) runner = if runner_registration_token_valid? diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 5270108ef0f..1d7126a432d 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -13,7 +13,6 @@ module Ci collect end - def push(from, to, format) @labels << from.strftime(format) @total << project.builds. diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e0b89cead06..01ef13df57a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,18 +2,18 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::ValidationHelpers + include Gitlab::Ci::Config::Node::LegacyValidationHelpers - DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables] + :dependencies, :before_script, :after_script, :variables, + :environment] ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :after_script, :image, :services, :path, :cache + attr_reader :path, :cache, :stages def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -21,15 +21,21 @@ module Ci @path = path - initial_parsing + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + initial_parsing validate! rescue Gitlab::Ci::Config::Loader::FormatError => e raise ValidationError, e.message end def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag, trigger_request)} + builds.select do |build| + build[:stage] == stage && + process?(build[:only], build[:except], ref, tag, trigger_request) + end end def builds @@ -38,10 +44,6 @@ module Ci end end - def stages - @stages || DEFAULT_STAGES - end - def global_variables @variables end @@ -50,18 +52,20 @@ module Ci job = @jobs[name.to_sym] return [] unless job - job.fetch(:variables, []) + job[:variables] || [] end private def initial_parsing - @after_script = @config[:after_script] - @image = @config[:image] - @services = @config[:services] - @stages = @config[:stages] || @config[:types] - @variables = @config[:variables] || {} - @cache = @config[:cache] + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache + @jobs = {} @config.except!(*ALLOWED_YAML_KEYS) @@ -81,15 +85,21 @@ module Ci def build_job(name, job) { - stage_idx: stages.index(job[:stage]), + stage_idx: @stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), + ## + # Refactoring note: + # - before script behaves differently than after script + # - after script returns an array of commands + # - before script should be a concatenated command + commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', + environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, @@ -102,12 +112,6 @@ module Ci end def validate! - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end - - validate_global! - @jobs.each do |name, job| validate_job!(name, job) end @@ -115,50 +119,6 @@ module Ci true end - def validate_global! - unless @after_script.nil? || validate_array_of_strings(@after_script) - raise ValidationError, "after_script should be an array of strings" - end - - unless @image.nil? || @image.is_a?(String) - raise ValidationError, "image should be a string" - end - - unless @services.nil? || validate_array_of_strings(@services) - raise ValidationError, "services should be an array of strings" - end - - unless @stages.nil? || validate_array_of_strings(@stages) - raise ValidationError, "stages should be an array of strings" - end - - unless @variables.nil? || validate_variables(@variables) - raise ValidationError, "variables should be a map of key-value strings" - end - - validate_global_cache! if @cache - end - - def validate_global_cache! - @cache.keys.each do |key| - unless ALLOWED_CACHE_KEYS.include? key - raise ValidationError, "#{name} cache unknown parameter #{key}" - end - end - - if @cache[:key] && !validate_string(@cache[:key]) - raise ValidationError, "cache:key parameter should be a string" - end - - if @cache[:untracked] && !validate_boolean(@cache[:untracked]) - raise ValidationError, "cache:untracked parameter should be an boolean" - end - - if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) - raise ValidationError, "cache:paths parameter should be an array of strings" - end - end - def validate_job!(name, job) validate_job_name!(name) validate_job_keys!(name, job) @@ -199,12 +159,12 @@ module Ci raise ValidationError, "#{name} job: tags parameter should be an array of strings" end - if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings" + if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) + raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" end - if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings" + if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) + raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" end if job[:allow_failure] && !validate_boolean(job[:allow_failure]) @@ -214,6 +174,10 @@ module Ci if job[:when] && !job[:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end + + if job[:environment] && !validate_environment(job[:environment]) + raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" + end end def validate_job_script!(name, job) @@ -231,8 +195,8 @@ module Ci end def validate_job_stage!(name, job) - unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" end end @@ -296,12 +260,12 @@ module Ci raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end - stage_index = stages.index(job[:stage]) + stage_index = @stages.index(job[:stage]) job[:dependencies].each do |dependency| raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index 4e20dc4f875..eb5a2596177 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -18,7 +18,7 @@ module ContainerRegistry end def digest - config['digest'] + config['digest'] || config['blobSum'] end def type diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 4d726692f45..42232b7129d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,11 +15,11 @@ module ContainerRegistry end def repository_tags(name) - @faraday.get("/v2/#{name}/tags/list").body + response_body @faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - @faraday.get("/v2/#{name}/manifests/#{reference}").body + response_body @faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) @@ -34,7 +34,7 @@ module ContainerRegistry def blob(name, digest, type = nil) headers = {} headers['Accept'] = type if type - @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) end def delete_blob(name, digest) @@ -47,7 +47,10 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION - conn.response :json, content_type: /\bjson$/ + conn.response :json, content_type: 'application/json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' if options[:user] && options[:password] conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) @@ -57,5 +60,9 @@ module ContainerRegistry conn.adapter :net_http end + + def response_body(response) + response.body if response.success? + end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 43f8d6dc8c2..708d01b95a1 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -3,6 +3,7 @@ module ContainerRegistry attr_reader :repository, :name delegate :registry, :client, to: :repository + delegate :revision, :short_revision, to: :config_blob, allow_nil: true def initialize(repository, name) @repository, @name = repository, name @@ -12,6 +13,14 @@ module ContainerRegistry manifest.present? end + def v1? + manifest && manifest['schemaVersion'] == 1 + end + + def v2? + manifest && manifest['schemaVersion'] == 2 + end + def manifest return @manifest if defined?(@manifest) @@ -57,7 +66,9 @@ module ContainerRegistry return @layers if defined?(@layers) return unless manifest - @layers = manifest['layers'].map do |layer| + layers = manifest['layers'] || manifest['fsLayers'] + + @layers = layers.map do |layer| repository.blob(layer) end end @@ -65,7 +76,7 @@ module ContainerRegistry def total_size return unless layers - layers.map(&:size).sum + layers.map(&:size).sum if v2? end def delete diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb index 1b80be112a4..cee664b8951 100644 --- a/lib/disable_email_interceptor.rb +++ b/lib/disable_email_interceptor.rb @@ -1,6 +1,5 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class DisableEmailInterceptor - def self.delivering_email(message) message.perform_deliveries = false Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 37f4c34054f..c3064163e07 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -2,6 +2,11 @@ require_dependency 'gitlab/git' module Gitlab def self.com? - Gitlab.config.gitlab.url == 'https://gitlab.com' + # Check `staging?` as well to keep parity with gitlab.com + Gitlab.config.gitlab.url == 'https://gitlab.com' || staging? + end + + def self.staging? + Gitlab.config.gitlab.url == 'https://staging.gitlab.com' end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6d0e30e916f..831f1e635ba 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -5,6 +5,8 @@ # module Gitlab module Access + class AccessDeniedError < StandardError; end + GUEST = 10 REPORTER = 20 DEVELOPER = 30 diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 0b9c2e730f9..1a22ad9acf5 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,7 +4,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index 51b1df9ecbd..c94bfc0e65f 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -66,8 +66,17 @@ module Gitlab def self.urls @urls ||= begin path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + # Construct the full asset path ourselves because + # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds + # of entries since it has to do a lot of extra work (e.g. regexps). prefix = Gitlab::Application.config.assets.prefix digest = Gitlab::Application.config.assets.digest + base = + if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root + Gitlab::Application.config.relative_url_root + else + '' + end JSON.parse(File.read(path)).map do |hash| if digest @@ -76,7 +85,7 @@ module Gitlab fname = hash['unicode'] end - { name: hash['name'], path: "#{prefix}/#{fname}.png" } + { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index adbf5941a96..478f145bfed 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,5 +1,3 @@ -require_relative 'shell_env' - module Grack class AuthSpawner def self.call(env) @@ -10,7 +8,6 @@ module Grack end class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env def call(env) @@ -24,7 +21,7 @@ module Grack # Need this if under RELATIVE_URL_ROOT unless Gitlab.config.gitlab.relative_url_root.empty? # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'') + @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') else @env['PATH_INFO'] = @request.path end @@ -33,7 +30,7 @@ module Grack auth! - lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call return lfs_response unless lfs_response.nil? if @user.nil? && !@ci @@ -61,11 +58,6 @@ module Grack end @user = authenticate_user(login, password) - - if @user - Gitlab::ShellEnv.set_env(@user) - @env['REMOTE_USER'] = @auth.username - end end def ci_request?(login, password) diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb deleted file mode 100644 index 9f5adee594a..00000000000 --- a/lib/gitlab/backend/shell_env.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - # This module provide 2 methods - # to set specific ENV variables for GitLab Shell - module ShellEnv - extend self - - def set_env(user) - # Set GL_ID env variable - if user - ENV['GL_ID'] = gl_id(user) - end - end - - def reset_env - # Reset GL_ID env variable - ENV['GL_ID'] = nil - end - - def gl_id(user) - if user.present? - "user-#{user.id}" - else - # This empty string is used in the render_grack_auth_ok method - "" - end - end - end -end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 997a22779a0..d62bc50ce78 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -41,7 +41,8 @@ module Gitlab def highlighted_lines @blob.load_all_data!(repository) - @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + @highlighted_lines ||= + Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines end def project diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index b48d3592f16..e6cc1529760 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,12 +4,11 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - delegate :valid?, :errors, to: :@global - ## # Temporary delegations that should be removed after refactoring # - delegate :before_script, to: :@global + delegate :before_script, :image, :services, :after_script, :variables, + :stages, :cache, to: :@global def initialize(config) @config = Loader.new(config).load! @@ -18,6 +17,14 @@ module Gitlab @global.process! end + def valid? + @global.valid? + end + + def errors + @global.errors + end + def to_hash @config end diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/node/boolean.rb new file mode 100644 index 00000000000..84b03ee7832 --- /dev/null +++ b/lib/gitlab/ci/config/node/boolean.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a boolean value. + # + class Boolean < Entry + include Validatable + + validations do + validates :config, boolean: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/node/cache.rb new file mode 100644 index 00000000000..cdf8ba2e35d --- /dev/null +++ b/lib/gitlab/ci/config/node/cache.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a cache configuration + # + class Cache < Entry + include Configurable + + node :key, Node::Key, + description: 'Cache key used to define a cache affinity.' + + node :untracked, Node::Boolean, + description: 'Cache all untracked files.' + + node :paths, Node::Paths, + description: 'Specify which paths should be cached across builds.' + + validations do + validates :config, allowed_keys: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index d60f87f3f94..37936fc8242 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -15,43 +15,49 @@ module Gitlab # module Configurable extend ActiveSupport::Concern + include Validatable - def allowed_nodes - self.class.allowed_nodes || {} + included do + validations do + validates :config, type: Hash + end end private - def prevalidate! - unless @value.is_a?(Hash) - @errors << 'should be a configuration entry with hash value' - end - end - def create_node(key, factory) - factory.with(value: @value[key]) - factory.nullify! unless @value.has_key?(key) + factory.with(value: @config[key], key: key, parent: self) + factory.create! end class_methods do - def allowed_nodes - Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + def nodes + Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end private - def allow_node(symbol, entry_class, metadata) + def node(symbol, entry_class, metadata) factory = Node::Factory.new(entry_class) .with(description: metadata[:description]) - define_method(symbol) do - raise Entry::InvalidError unless valid? + (@nodes ||= {}).merge!(symbol.to_sym => factory) + end - @nodes[symbol].try(:value) - end + def helpers(*nodes) + nodes.each do |symbol| + define_method("#{symbol}_defined?") do + @nodes[symbol].try(:defined?) + end - (@allowed_nodes ||= {}).merge!(symbol => factory) + define_method("#{symbol}_value") do + raise Entry::InvalidError unless valid? + @nodes[symbol].try(:value) + end + + alias_method symbol.to_sym, "#{symbol}_value".to_sym + end end end end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 52758a962f3..9e79e170a4f 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -8,14 +8,14 @@ module Gitlab class Entry class InvalidError < StandardError; end - attr_accessor :description + attr_reader :config + attr_accessor :key, :parent, :description - def initialize(value) - @value = value + def initialize(config) + @config = config @nodes = {} - @errors = [] - - prevalidate! + @validator = self.class.validator.new(self) + @validator.validate end def process! @@ -23,50 +23,65 @@ module Gitlab return unless valid? compose! - - nodes.each(&:process!) - nodes.each(&:validate!) + process_nodes! end def nodes @nodes.values end - def valid? - errors.none? + def leaf? + self.class.nodes.none? end - def leaf? - allowed_nodes.none? + def ancestors + @parent ? @parent.ancestors + [@parent] : [] + end + + def valid? + errors.none? end def errors - @errors + nodes.map(&:errors).flatten + @validator.messages + nodes.flat_map(&:errors) end - def allowed_nodes - {} + def value + if leaf? + @config + else + defined = @nodes.select { |_key, value| value.defined? } + Hash[defined.map { |key, node| [key, node.value] }] + end end - def validate! - raise NotImplementedError + def defined? + true end - def value - raise NotImplementedError + def self.default end - private + def self.nodes + {} + end - def prevalidate! + def self.validator + Validator end + private + def compose! - allowed_nodes.each do |key, essence| + self.class.nodes.each do |key, essence| @nodes[key] = create_node(key, essence) end end + def process_nodes! + nodes.each(&:process!) + end + def create_node(key, essence) raise NotImplementedError end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 787ca006f5a..5919a283283 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -5,13 +5,11 @@ module Gitlab ## # Factory class responsible for fabricating node entry objects. # - # It uses Fluent Interface pattern to set all necessary attributes. - # class Factory class InvalidFactory < StandardError; end - def initialize(entry_class) - @entry_class = entry_class + def initialize(node) + @node = node @attributes = {} end @@ -20,18 +18,29 @@ module Gitlab self end - def nullify! - @entry_class = Node::Null - self - end - def create! raise InvalidFactory unless @attributes.has_key?(:value) - @entry_class.new(@attributes[:value]).tap do |entry| + fabricate.tap do |entry| + entry.key = @attributes[:key] + entry.parent = @attributes[:parent] entry.description = @attributes[:description] end end + + private + + def fabricate + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @attributes[:value].nil? + Node::Undefined.new(@node) + else + @node.new(@attributes[:value]) + end + end end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index 044603423d5..f92e1eccbcf 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -9,8 +9,36 @@ module Gitlab class Global < Entry include Configurable - allow_node :before_script, Script, + node :before_script, Node::Script, description: 'Script that will be executed before each job.' + + node :image, Node::Image, + description: 'Docker image that will be used to execute jobs.' + + node :services, Node::Services, + description: 'Docker images that will be linked to the container.' + + node :after_script, Node::Script, + description: 'Script that will be executed after each job.' + + node :variables, Node::Variables, + description: 'Environment variables that will be used.' + + node :stages, Node::Stages, + description: 'Configuration of stages for this pipeline.' + + node :types, Node::Stages, + description: 'Deprecated: stages for this pipeline.' + + node :cache, Node::Cache, + description: 'Configure caching between build jobs.' + + helpers :before_script, :image, :services, :after_script, + :variables, :stages, :types, :cache + + def stages + stages_defined? ? stages_value : types_value + end end end end diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/node/image.rb new file mode 100644 index 00000000000..5d3c7c5eab0 --- /dev/null +++ b/lib/gitlab/ci/config/node/image.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a Docker image. + # + class Image < Entry + include Validatable + + validations do + validates :config, type: String + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/node/key.rb new file mode 100644 index 00000000000..f8b461ca098 --- /dev/null +++ b/lib/gitlab/ci/config/node/key.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a key. + # + class Key < Entry + include Validatable + + validations do + validates :config, key: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 42ef60244ba..4d9a508796a 100644 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -2,7 +2,7 @@ module Gitlab module Ci class Config module Node - module ValidationHelpers + module LegacyValidationHelpers private def validate_duration(value) @@ -15,6 +15,10 @@ module Gitlab values.is_a?(Array) && values.all? { |value| validate_string(value) } end + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + def validate_variables(variables) variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } @@ -24,6 +28,23 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + Regexp.new(value[1...-1]) + else + true + end + rescue RegexpError + false + end + + def validate_environment(value) + value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex + end + def validate_boolean(value) value.in?([true, false]) end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 4f590f6bec8..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents a configuration entry that is not being used - # in configuration file. - # - # This implements Null Object pattern. - # - class Null < Entry - def value - nil - end - - def validate! - nil - end - - def method_missing(*) - nil - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/node/paths.rb new file mode 100644 index 00000000000..3c6d3a52966 --- /dev/null +++ b/lib/gitlab/ci/config/node/paths.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an array of paths. + # + class Paths < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb index 5072bf0db7d..39328f0fade 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/node/script.rb @@ -5,22 +5,11 @@ module Gitlab ## # Entry that represents a script. # - # Each element in the value array is a command that will be executed - # by GitLab Runner. Currently we concatenate these commands with - # new line character as a separator, what is compatible with - # implementation in Runner. - # class Script < Entry - include ValidationHelpers - - def value - @value.join("\n") - end + include Validatable - def validate! - unless validate_array_of_strings(@value) - @errors << 'before_script should be an array of strings' - end + validations do + validates :config, array_of_strings: true end end end diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/node/services.rb new file mode 100644 index 00000000000..481e2b66adc --- /dev/null +++ b/lib/gitlab/ci/config/node/services.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration of Docker services. + # + class Services < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/node/stages.rb new file mode 100644 index 00000000000..b1fe45357ff --- /dev/null +++ b/lib/gitlab/ci/config/node/stages.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration for pipeline stages. + # + class Stages < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + + def self.default + %w[build test deploy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb new file mode 100644 index 00000000000..699605e1e3a --- /dev/null +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an undefined entry node. + # + # It takes original entry class as configuration and returns default + # value of original entry as self value. + # + # + class Undefined < Entry + include Validatable + + validations do + validates :config, type: Class + end + + def value + @config.default + end + + def defined? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb new file mode 100644 index 00000000000..f6e2896dfb2 --- /dev/null +++ b/lib/gitlab/ci/config/node/validatable.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + module Validatable + extend ActiveSupport::Concern + + class_methods do + def validator + validator = Class.new(Node::Validator) + + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end + + validator + end + + private + + def validations(&block) + (@validations ||= []).append(block) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb new file mode 100644 index 00000000000..758a6cf4356 --- /dev/null +++ b/lib/gitlab/ci/config/node/validator.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + class Config + module Node + class Validator < SimpleDelegator + include ActiveModel::Validations + include Node::Validators + + def initialize(node) + super(node) + @node = node + end + + def messages + errors.full_messages.map do |error| + "#{location} #{error}".downcase + end + end + + def self.name + 'Validator' + end + + def unknown_keys + return [] unless config.is_a?(Hash) + + config.keys - @node.class.nodes.keys + end + + private + + def location + predecessors = ancestors.map(&:key).compact + current = key || @node.class.name.demodulize.underscore + predecessors.append(current).join(':') + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb new file mode 100644 index 00000000000..7b2f57990b5 --- /dev/null +++ b/lib/gitlab/ci/config/node/validators.rb @@ -0,0 +1,70 @@ +module Gitlab + module Ci + class Config + module Node + module Validators + class AllowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + if record.unknown_keys.any? + unknown_list = record.unknown_keys.join(', ') + record.errors.add(:config, + "contains unknown keys: #{unknown_list}") + end + end + end + + class ArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_array_of_strings(value) + record.errors.add(attribute, 'should be an array of strings') + end + end + end + + class BooleanValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_boolean(value) + record.errors.add(attribute, 'should be a boolean value') + end + end + end + + class KeyValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_string(value) + record.errors.add(attribute, 'should be a string or symbol') + end + end + end + + class TypeValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + type = options[:with] + raise unless type.is_a?(Class) + + unless value.is_a?(type) + record.errors.add(attribute, "should be a #{type.name}") + end + end + end + + class VariablesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs') + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/node/variables.rb new file mode 100644 index 00000000000..5f813f81f55 --- /dev/null +++ b/lib/gitlab/ci/config/node/variables.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents environment variables. + # + class Variables < Entry + include Validatable + + validations do + validates :config, variables: true + end + + def self.default + {} + end + end + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 5e7532f57ae..ffc1814b29d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,10 +9,14 @@ module Gitlab end def ensure_application_settings! - settings = ::ApplicationSetting.cached + if connect_to_db? + begin + settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + settings = ::ApplicationSetting.last + end - if !settings && connect_to_db? - settings = ::ApplicationSetting.current settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end @@ -36,7 +40,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, @@ -44,6 +48,7 @@ module Gitlab akismet_enabled: false, repository_checks_enabled: true, container_registry_token_expire_delay: 5, + user_default_external: false, ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04fa6a3a5de..078609c86f1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,5 +1,10 @@ module Gitlab module Database + # The max value of INTEGER type is the same between MySQL and PostgreSQL: + # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html + # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html + MAX_INT_VALUE = 2147483647 + def self.adapter_name connection.adapter_name end @@ -30,6 +35,10 @@ module Gitlab order end + def self.random + Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + end + def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dd3ff0ab18b..dec20d8659b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -28,65 +28,79 @@ module Gitlab # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. - # Any data inserted while running this method (or after it has finished - # running) is _not_ updated automatically. + # This method will continue updating rows until no rows remain. + # + # When given a block this method will yield two values to the block: + # + # 1. An instance of `Arel::Table` for the table that is being updated. + # 2. The query to run as an Arel object. + # + # By supplying a block one can add extra conditions to the queries being + # executed. Note that the same block is used for _all_ queries. + # + # Example: + # + # update_column_in_batches(:projects, :foo, 10) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # + # This would result in this method updating only rows where + # `projects.some_column` equals "hello". # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. + # + # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop + # determines this method to be too complex while there's no way to make it + # less "complex" without introducing extra methods (which actually will + # make things _more_ complex). + # + # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) - quoted_table = quote_table_name(table) - quoted_column = quote_column_name(column) - - ## - # Workaround for #17711 - # - # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` - # returns correct value (1), but `ActiveRecord::Migration.new.quote` - # returns incorrect value ('true'), which causes migrations to fail. - # - quoted_value = connection.quote(value) - processed = 0 - - total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). - to_hash. - first['count']. - to_i + table = Arel::Table.new(table) + + count_arel = table.project(Arel.star.count.as('count')) + count_arel = yield table, count_arel if block_given? + + total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + + return if total == 0 # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil + start_arel = table.project(table[:id]).order(table[:id].asc).take(1) + start_arel = yield table, start_arel if block_given? + start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i + loop do - start_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed} - }).to_hash.first - - # There are no more rows to process - break unless start_row - - stop_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed + batch_size} - }).to_hash.first - - query = %Q{ - UPDATE #{quoted_table} - SET #{quoted_column} = #{quoted_value} - WHERE id >= #{start_row['id']} - } + stop_arel = table.project(table[:id]). + where(table[:id].gteq(start_id)). + order(table[:id].asc). + take(1). + skip(batch_size) + + stop_arel = yield table, stop_arel if block_given? + stop_row = exec_query(stop_arel.to_sql).to_hash.first + + update_arel = Arel::UpdateManager.new(ActiveRecord::Base). + table(table). + set([[table[column], value]]). + where(table[:id].gteq(start_id)) if stop_row - query += " AND id < #{stop_row['id']}" + stop_id = stop_row['id'].to_i + start_id = stop_id + update_arel = update_arel.where(table[:id].lt(stop_id)) end - execute(query) + update_arel = yield table, update_arel if block_given? + + execute(update_arel.to_sql) - processed += batch_size + # There are no more rows left to update. + break unless stop_row end end @@ -95,9 +109,9 @@ module Gitlab # This method runs the following steps: # # 1. Add the column with a default value of NULL. - # 2. Update all existing rows in batches. - # 3. Change the default value of the column to the specified value. - # 4. Update any remaining rows. + # 2. Change the default value of the column to the specified value. + # 3. Update all existing rows in batches. + # 4. Set a `NOT NULL` constraint on the column if desired (the default). # # These steps ensure a column can be added to a large and commonly used # table without locking the entire table for the duration of the table @@ -109,7 +123,10 @@ module Gitlab # default - The default value for the column. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. - def add_column_with_default(table, column, type, default:, allow_null: false) + # + # This method can also take a block which is passed directly to the + # `update_column_in_batches` method. + def add_column_with_default(table, column, type, default:, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -125,11 +142,9 @@ module Gitlab end begin - transaction do - update_column_in_batches(table, column, default) + update_column_in_batches(table, column, default, &block) - change_column_null(table, column, false) unless allow_null - end + change_column_null(table, column, false) unless allow_null # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. rescue Exception => error # rubocop: disable all diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb new file mode 100644 index 00000000000..8406ca4269c --- /dev/null +++ b/lib/gitlab/diff/diff_refs.rb @@ -0,0 +1,36 @@ +module Gitlab + module Diff + class DiffRefs + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(base_sha:, start_sha: base_sha, head_sha:) + @base_sha = base_sha + @start_sha = start_sha + @head_sha = head_sha + end + + def ==(other) + other.is_a?(self.class) && + base_sha == other.base_sha && + start_sha == other.start_sha && + head_sha == other.head_sha + end + + # There is only one case in which we will have `start_sha` and `head_sha`, + # but not `base_sha`, which is when a diff is generated between an + # orphaned branch and another branch, which means there _is_ no base, but + # we're still able to highlight it, and to create diff notes, which are + # the primary things `DiffRefs` are used for. + # `DiffRefs` are "complete" when they have `start_sha` and `head_sha`, + # because `base_sha` can always be derived from this, to return an actual + # sha, or `nil`. + # We have `base_sha` directly available on `DiffRefs` because it's faster# + # than having to look it up in the repo every time. + def complete? + start_sha && head_sha + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2e85cabf72..7e01f7b61fb 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,47 +1,87 @@ module Gitlab module Diff class File - attr_reader :diff, :diff_refs + attr_reader :diff, :repository, :diff_refs delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, to: :diff, prefix: false + :old_path, :new_path, :a_mode, :b_mode, + :submodule?, :too_large?, to: :diff, prefix: false - def initialize(diff, diff_refs) + def initialize(diff, repository:, diff_refs: nil) @diff = diff + @repository = repository @diff_refs = diff_refs end + def position(line) + return unless diff_refs + + Position.new( + old_path: old_path, + new_path: new_path, + old_line: line.old_line, + new_line: line.new_line, + diff_refs: diff_refs + ) + end + + def line_code(line) + return if line.meta? + + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def line_for_line_code(code) + diff_lines.find { |line| line_code(line) == code } + end + + def line_for_position(pos) + diff_lines.find { |line| position(line) == pos } + end + + def position_for_line_code(code) + line = line_for_line_code(code) + position(line) if line + end + + def line_code_for_position(pos) + line = line_for_position(pos) + line_code(line) if line + end + + def content_commit + return unless diff_refs + + repository.commit(deleted_file ? old_ref : new_ref) + end + def old_ref - diff_refs[0] if diff_refs + diff_refs.try(:base_sha) end def new_ref - diff_refs[1] if diff_refs + diff_refs.try(:head_sha) end - # Array of Gitlab::DIff::Line objects + # Array of Gitlab::Diff::Line objects def diff_lines - @lines ||= parser.parse(raw_diff.each_line).to_a + @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a end - def too_large? - diff.too_large? + def collapsed_by_default? + diff.diff.bytesize > 10240 # 10 KB end def highlighted_diff_lines - Gitlab::Diff::Highlight.new(self).highlight + @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end def parallel_diff_lines - Gitlab::Diff::ParallelDiff.new(self).parallelize + @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end def mode_changed? - !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) - end - - def parser - Gitlab::Diff::Parser.new + a_mode && b_mode && a_mode != b_mode end def raw_diff @@ -53,17 +93,15 @@ module Gitlab end def prev_line(index) - if index > 0 - diff_lines[index - 1] - end + diff_lines[index - 1] if index > 0 + end + + def paths + [old_path, new_path].compact end def file_path - if diff.new_path.present? - diff.new_path - elsif diff.old_path.present? - diff.old_path - end + new_path.presence || old_path end def added_lines @@ -73,6 +111,21 @@ module Gitlab def removed_lines diff_lines.count(&:removed?) end + + def old_blob(commit = content_commit) + return unless commit + + parent_id = commit.parent_id + return unless parent_id + + repository.blob_at(parent_id, old_path) + end + + def blob(commit = content_commit) + return unless commit + + repository.blob_at(commit.id, file_path) + end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 9429b3ff88d..649a265a02c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,11 +1,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines + attr_reader :diff_file, :diff_lines, :raw_lines, :repository delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - def initialize(diff_lines) + def initialize(diff_lines, repository: nil) + @repository = repository + if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @diff_lines = @diff_file.diff_lines @@ -19,7 +21,7 @@ module Gitlab @diff_lines.map.with_index do |diff_line, i| diff_line = diff_line.dup # ignore highlighting for "match" lines - next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + next diff_line if diff_line.meta? rich_line = highlight_line(diff_line) || diff_line.text @@ -40,12 +42,12 @@ module Gitlab line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' - case diff_line.type - when 'new', nil - rich_line = new_lines[diff_line.new_pos - 1] - when 'old' - rich_line = old_lines[diff_line.old_pos - 1] - end + rich_line = + if diff_line.unchanged? || diff_line.added? + new_lines[diff_line.new_pos - 1] + elsif diff_line.removed? + old_lines[diff_line.old_pos - 1] + end # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. @@ -58,19 +60,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) - end - - def processing_args(version) - ref = send("diff_#{version}_ref") - path = send("diff_#{version}_path") - - [ref.project.repository, ref.id, path] + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) end end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 789c14518b0..28ad637fda4 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,16 +1,30 @@ module Gitlab module Diff class InlineDiff + # Regex to find a run of deleted lines followed by the same number of added lines + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze + attr_accessor :old_line, :new_line, :offset def self.for_lines(lines) - local_edit_indexes = self.find_local_edits(lines) + changed_line_pairs = self.find_changed_line_pairs(lines) inline_diffs = [] - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 + changed_line_pairs.each do |old_index, new_index| old_line = lines[old_index] new_line = lines[new_index] @@ -51,18 +65,28 @@ module Gitlab private - def self.find_local_edits(lines) - line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - joined_line_prefixes = " #{line_prefixes.join} " - - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + # Finds pairs of old/new line pairs that represent the same line that changed + def self.find_changed_line_pairs(lines) + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + + changed_line_pairs = [] + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + changed_line_pairs << [i, i + changed_line_count] + end end - local_edit_indexes + changed_line_pairs end def longest_common_prefix(a, b) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 03730b435ad..c6189d660c2 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -9,6 +9,18 @@ module Gitlab @old_pos, @new_pos = old_pos, new_pos end + def old_line + old_pos unless added? || meta? + end + + def new_line + new_pos unless removed? || meta? + end + + def unchanged? + type.nil? + end + def added? type == 'new' end @@ -16,6 +28,10 @@ module Gitlab def removed? type == 'old' end + + def meta? + type == 'match' || type == 'nonewline' + end end end end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb new file mode 100644 index 00000000000..576a761423e --- /dev/null +++ b/lib/gitlab/diff/line_mapper.rb @@ -0,0 +1,64 @@ +# When provided a diff for a specific file, maps old line numbers to new line +# numbers and back, to find out where a specific line in a file was moved by the +# changes. +module Gitlab + module Diff + class LineMapper + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + # Find new line number for old line number. + def old_to_new(old_line) + map_line_number(old_line, from: :old_line, to: :new_line) + end + + # Find old line number for new line number. + def new_to_old(new_line) + map_line_number(new_line, from: :new_line, to: :old_line) + end + + private + + def diff_lines + @diff_lines ||= @diff_file.diff_lines + end + + # Find old/new line number based on its old/new counterpart line number. + def map_line_number(from_line, from:, to:) + # If no diff file could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_file + + # To find the mapped line number for the specified line number, + # we need to find: + # - The diff line with that exact line number, if it is in the diff context + # - The first diff line with a higher line number, if it falls between diff contexts + # - The last known diff line, if it falls after the last diff context + diff_line = diff_lines.find do |diff_line| + diff_from_line = diff_line.send(from) + diff_from_line && diff_from_line >= from_line + end + diff_line ||= diff_lines.last + + # If no diff line could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_line + + diff_from_line = diff_line.send(from) + diff_to_line = diff_line.send(to) + + # If the line was removed, there is no mapped line number. + return unless diff_to_line + + # Because we may not have the diff line with the exact line number + # we were looking for, we need to adjust the mapped line number. + distance = diff_from_line - from_line + + diff_to_line - distance + end + end + end +end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 74f9b3c050a..b069afdd28c 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -8,111 +8,96 @@ module Gitlab end def parallelize - lines = [] - skip_next = false + i = 0 + free_right_index = nil + + lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - full_line = line.text - type = line.type - line_code = generate_line_code(diff_file.file_path, line) - line_new = line.new_pos - line_old = line.old_pos + line_code = diff_file.line_code(line) + position = diff_file.position(line) - next_line = diff_file.next_line(line.index) - - if next_line - next_line = highlighted_diff_lines[next_line.index] - next_line_code = generate_line_code(diff_file.file_path, next_line) - next_type = next_line.type - next_line = next_line.text - end - - case type + case line.type when 'match', nil # line in the right panel is the same as in the left one lines << { left: { - type: type, - number: line_old, - text: full_line, + type: line.type, + number: line.old_pos, + text: line.text, line_code: line_code, + position: position }, right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code + type: line.type, + number: line.new_pos, + text: line.text, + line_code: line_code, + position: position } } + + free_right_index = nil + i += 1 when 'old' - case next_type - when 'new' - # Left side has text removed, right side has text added - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - }, - right: { - type: next_type, - number: line_new, - text: next_line, - line_code: next_line_code - } - } - skip_next = true - when 'old', 'nonewline', nil - # Left side has text removed, right side doesn't have any change - # No next line code, no new line number, no new line text - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - }, - right: { - type: next_type, - number: nil, - text: "", - line_code: nil - } + lines << { + left: { + type: line.type, + number: line.old_pos, + text: line.text, + line_code: line_code, + position: position + }, + right: { + type: nil, + number: nil, + text: "", + line_code: line_code, + position: position } - end + } + + # Once we come upon a new line it can be put on the right of this old line + free_right_index ||= i + i += 1 when 'new' - if skip_next - # Change has been already included in previous line so no need to do it again - skip_next = false - next + data = { + type: line.type, + number: line.new_pos, + text: line.text, + line_code: line_code, + position: position + } + + if free_right_index + # If an old line came before this without a line on the right, this + # line can be put to the right of it. + lines[free_right_index][:right] = data + + # If there are any other old lines on the left that don't yet have + # a new counterpart on the right, update the free_right_index + next_free_right_index = free_right_index + 1 + free_right_index = next_free_right_index < i ? next_free_right_index : nil else - # Change is only on the right side, left side has no change lines << { left: { type: nil, number: nil, text: "", line_code: line_code, + position: position }, - right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code - } + right: data } + + free_right_index = nil + i += 1 end end end - lines - end - private - - def generate_line_code(file_path, line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + lines end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 522dd2b9428..59a2367b65d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -40,7 +40,6 @@ module Gitlab line_obj_index += 1 end - case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb new file mode 100644 index 00000000000..989fff8918e --- /dev/null +++ b/lib/gitlab/diff/position.rb @@ -0,0 +1,155 @@ +# Defines a specific location, identified by paths and line numbers, +# within a specific diff, identified by start, head and base commit ids. +module Gitlab + module Diff + class Position + attr_reader :old_path + attr_reader :new_path + attr_reader :old_line + attr_reader :new_line + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(attrs = {}) + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + if attrs[:diff_refs] + @base_sha = attrs[:diff_refs].base_sha + @start_sha = attrs[:diff_refs].start_sha + @head_sha = attrs[:diff_refs].head_sha + else + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + end + + # `Gitlab::Diff::Position` objects are stored as serialized attributes in + # `DiffNote`, which use YAML to encode and decode objects. + # `#init_with` and `#encode_with` can be used to customize the en/decoding + # behavior. In this case, we override these to prevent memoized instance + # variables like `@diff_file` and `@diff_line` from being serialized. + def init_with(coder) + initialize(coder['attributes']) + + self + end + + def encode_with(coder) + coder['attributes'] = self.to_h + end + + def key + @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + end + + def ==(other) + other.is_a?(self.class) && key == other.key + end + + def to_h + { + old_path: old_path, + new_path: new_path, + old_line: old_line, + new_line: new_line, + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha + } + end + + def inspect + %(#<#{self.class}:#{object_id} #{to_h}>) + end + + def complete? + file_path.present? && + (old_line || new_line) && + diff_refs.complete? + end + + def to_json + JSON.generate(self.to_h) + end + + def type + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def unchanged? + type.nil? + end + + def added? + type == 'new' + end + + def removed? + type == 'old' + end + + def paths + [old_path, new_path].compact.uniq + end + + def file_path + new_path.presence || old_path + end + + def diff_refs + @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) + end + + def diff_file(repository) + @diff_file ||= begin + if RequestStore.active? + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + RequestStore.fetch(key) { find_diff_file(repository) } + else + find_diff_file(repository) + end + end + end + + def diff_line(repository) + @diff_line ||= diff_file(repository).line_for_position(self) + end + + def line_code(repository) + @line_code ||= diff_file(repository).line_code_for_position(self) + end + + private + + def find_diff_file(repository) + diffs = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ).diffs(paths: paths) + + diff = diffs.first + return unless diff + + Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb new file mode 100644 index 00000000000..4d04f867268 --- /dev/null +++ b/lib/gitlab/diff/position_tracer.rb @@ -0,0 +1,168 @@ +# Finds the diff position in the new diff that corresponds to the same location +# specified by the provided position in the old diff. +module Gitlab + module Diff + class PositionTracer + attr_accessor :repository + attr_accessor :old_diff_refs + attr_accessor :new_diff_refs + attr_accessor :paths + + def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) + @repository = repository + @old_diff_refs = old_diff_refs + @new_diff_refs = new_diff_refs + @paths = paths + end + + def trace(old_position) + return unless old_diff_refs.complete? && new_diff_refs.complete? + return unless old_position.diff_refs == old_diff_refs + + # Suppose we have an MR with source branch `feature` and target branch `master`. + # When the MR was created, the head of `master` was commit A, and the + # head of `feature` was commit B, resulting in the original diff A->B. + # Since creation, `master` was updated to C. + # Now `feature` is being updated to D, and the newly generated MR diff is C->D. + # It is possible that C and D are direct decendants of A and B respectively, + # but this isn't necessarily the case as rebases and merges come into play. + # + # Suppose we have a diff note on the original diff A->B. Now that the MR + # is updated, we need to find out what line in C->D corresponds to the + # line the note was originally created on, so that we can update the diff note's + # records and continue to display it in the right place in the diffs. + # If we cannot find this line in the new diff, this means the diff note is now + # outdated, and we will display that fact to the user. + # + # In the new diff, the file the diff note was originally created on may + # have been renamed, deleted or even created, if the file existed in A and B, + # but was removed in C, and restored in D. + # + # Every diff note stores a Position object that defines a specific location, + # identified by paths and line numbers, within a specific diff, identified + # by start, head and base commit ids. + # + # For diff notes for diff A->B, the position looks like this: + # Position + # base_sha - ID of commit A + # head_sha - ID of commit B + # old_path - path as of A (nil if file was newly created) + # new_path - path as of B (nil if file was deleted) + # old_line - line number as of A (nil if file was newly created) + # new_line - line number as of B (nil if file was deleted) + # + # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # but need to find the paths and line numbers as of C and D. + # + # If the file was unchanged or newly created in A->B, the path as of D can be found + # by generating diff B->D ("head to head"), finding the diff file with + # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. + # The path as of C can be found by taking diff C->D, finding the diff file + # with that same `new_path` and taking `diff_file.old_path`. + # The line number as of D can be found by using the LineMapper on diff B->D + # and providing the line number as of B. + # The line number as of C can be found by using the LineMapper on diff C->D + # and providing the line number as of D. + # + # If the file was deleted in A->B, the path as of C can be found + # by generating diff A->C ("base to base"), finding the diff file with + # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. + # The path as of D can be found by taking diff C->D, finding the diff file + # with that same `old_path` and taking `diff_file.new_path`. + # The line number as of C can be found by using the LineMapper on diff A->C + # and providing the line number as of A. + # The line number as of D can be found by using the LineMapper on diff C->D + # and providing the line number as of C. + + results = nil + results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? + results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? + + return unless results + + file_diff, old_line, new_line = results + + Position.new( + old_path: file_diff.old_path, + new_path: file_diff.new_path, + head_sha: new_diff_refs.head_sha, + start_sha: new_diff_refs.start_sha, + base_sha: new_diff_refs.base_sha, + old_line: old_line, + new_line: new_line + ) + end + + private + + def trace_added_line(old_position) + file_path = old_position.new_path + + return unless diff_head_to_head + + file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_head_to_head.new_path if file_head_to_head + + new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) + + return unless new_line + + file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } + return unless file_diff + + old_line = LineMapper.new(file_diff).new_to_old(new_line) + + [file_diff, old_line, new_line] + end + + def trace_removed_line(old_position) + file_path = old_position.old_path + + return unless diff_base_to_base + + file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_base_to_base.old_path if file_base_to_base + + old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + + return unless old_line + + file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } + return unless file_diff + + new_line = LineMapper.new(file_diff).old_to_new(old_line) + + [file_diff, old_line, new_line] + end + + def diff_base_to_base + @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + end + + def diff_head_to_head + @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + end + + def new_diffs + @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + end + + def diff_files(start_sha, head_sha, use_base: false) + base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + + diffs = self.repository.raw_repository.diff( + use_base ? base_sha : start_sha, + head_sha, + {}, + *paths + ) + + diffs.decorate! do |diff| + Gitlab::Diff::File.new(diff, repository: self.repository) + end + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index e2fee6b9f3e..97701b0cd42 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -33,11 +33,15 @@ module Gitlab end def commits - @commits ||= (Commit.decorate(compare.commits, project) if compare) + return unless compare + + @commits ||= Commit.decorate(compare.commits, project) end def diffs - @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) + return unless compare + + @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository) end def diffs_count diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb new file mode 100644 index 00000000000..b63213ae208 --- /dev/null +++ b/lib/gitlab/emoji.rb @@ -0,0 +1,21 @@ +module Gitlab + module Emoji + extend self + + def emojis + Gemojione.index.instance_variable_get(:@emoji_by_name) + end + + def emojis_by_moji + Gemojione.index.instance_variable_get(:@emoji_by_moji) + end + + def emojis_names + emojis.keys.sort + end + + def emoji_filename(name) + emojis[name]["unicode"] + end + end +end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 07b856ca64c..9b681e636c7 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,6 +1,7 @@ module Gitlab module Git class Hook + GL_PROTOCOL = 'web'.freeze attr_reader :name, :repo_path, :path def initialize(name, repo_path) @@ -14,7 +15,7 @@ module Gitlab end def trigger(gl_id, oldrev, newrev, ref) - return true unless exists? + return [true, nil] unless exists? case name when "pre-receive", "post-receive" @@ -29,19 +30,20 @@ module Gitlab def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") - # function will return true if succesful exit_status = false + exit_message = nil vars = { 'GL_ID' => gl_id, - 'PWD' => repo_path + 'PWD' => repo_path, + 'GL_PROTOCOL' => GL_PROTOCOL } options = { chdir: repo_path } - Open3.popen2(vars, path, options) do |stdin, _, wait_thr| + Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| exit_status = true stdin.sync = true @@ -60,17 +62,24 @@ module Gitlab unless wait_thr.value == 0 exit_status = false + exit_message = retrieve_error_message(stderr, stdout) end end - exit_status + [exit_status, exit_message] end def call_update_hook(gl_id, oldrev, newrev, ref) Dir.chdir(repo_path) do - system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + [status.success?, stderr.presence || stdout] end end + + def retrieve_error_message(stderr, stdout) + err_message = stderr.gets + err_message.blank? ? stdout.gets : err_message + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index d2a0e316cbe..7679c7e4bb8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -3,11 +3,12 @@ module Gitlab DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project + attr_reader :actor, :project, :protocol - def initialize(actor, project) + def initialize(actor, project, protocol) @actor = actor @project = project + @protocol = protocol end def user @@ -49,6 +50,8 @@ module Gitlab end def check(cmd, changes = nil) + return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? + unless actor return build_status_object(false, "No user or key was provided.") end @@ -164,6 +167,10 @@ module Gitlab Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) end + def protocol_allowed? + Gitlab::ProtocolAccess.allowed?(protocol) + end + private def protected_branch_action(oldrev, newrev, branch_name) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index a15fc84b418..7d2d545b84e 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -4,7 +4,7 @@ module Gitlab delegate :repo, :sha, :ref, to: :raw_data def exists? - project.repository.branch_exists?(ref) + branch_exists? && commit_exists? end def name @@ -15,11 +15,15 @@ module Gitlab repo.present? end - def valid? - repo.present? + private + + def branch_exists? + project.repository.branch_exists?(ref) end - private + def commit_exists? + project.repository.commit(sha).present? + end def short_id sha.to_s[0..7] diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..084e514492c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,15 +69,30 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? end def rate_limit_exceed? - rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS end def rate_limit_sleep_time diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index e5cf66a0371..3932fcb1eda 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -66,8 +66,7 @@ module Gitlab end def import_pull_requests - hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - disable_webhooks(hooks) + disable_webhooks pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) @@ -90,14 +89,14 @@ module Gitlab raise Projects::ImportService::Error, e.message ensure clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks(hooks) + clean_up_disabled_webhooks end - def disable_webhooks(hooks) + def disable_webhooks update_webhooks(hooks, active: false) end - def clean_up_disabled_webhooks(hooks) + def clean_up_disabled_webhooks update_webhooks(hooks, active: true) end @@ -107,6 +106,20 @@ module Gitlab end end + def hooks + @hooks ||= + begin + client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + + # The GitHub Repository Webhooks API returns 404 for users + # without admin access to the repository when listing hooks. + # In this case we just want to return gracefully instead of + # spitting out an error and stop the import process. + rescue Octokit::NotFound + [] + end + end + def restore_branches(branches) branches.each do |name, sha| client.create_ref(repo, "refs/heads/#{name}", sha) @@ -118,8 +131,10 @@ module Gitlab def clean_up_restored_branches(branches) branches.each do |name, _| client.delete_ref(repo, "heads/#{name}") - project.repository.rm_branch(project.creator, name) + project.repository.delete_branch(name) rescue Rugged::ReferenceError end + + project.repository.after_remove_branch end def apply_labels(issuable) @@ -154,7 +169,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 498b00cb658..a4ea2210abd 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -11,10 +11,10 @@ module Gitlab description: description, source_project: source_branch_project, source_branch: source_branch_name, - head_source_sha: source_branch_sha, + source_branch_sha: source_branch_sha, target_project: target_branch_project, target_branch: target_branch_name, - base_target_sha: target_branch_sha, + target_branch_sha: target_branch_sha, state: state, milestone: milestone, author_id: author_id, diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb deleted file mode 100644 index f46b43b61a4..00000000000 --- a/lib/gitlab/gitignore.rb +++ /dev/null @@ -1,56 +0,0 @@ -module Gitlab - class Gitignore - FILTER_REGEX = /\.gitignore\z/.freeze - - def initialize(path) - @path = path - end - - def name - File.basename(@path, '.gitignore') - end - - def content - File.read(@path) - end - - class << self - def all - languages_frameworks + global - end - - def find(key) - file_name = "#{key}.gitignore" - - directory = select_directory(file_name) - directory ? new(File.join(directory, file_name)) : nil - end - - def global - files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } - end - - def languages_frameworks - files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } - end - - private - - def select_directory(file_name) - [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } - end - - def global_dir - File.join(gitignore_dir, 'Global') - end - - def gitignore_dir - Rails.root.join('vendor/gitignore') - end - - def files_for_folder(dir) - Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } - end - end - end -end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 3f76ec97977..e6d31ea04c0 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -35,6 +35,7 @@ module Gitlab end project.issues.create!( + iid: issue["iid"], description: body, title: issue["title"], state: issue["state"], diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 77c33db4b59..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,8 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project end end end diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb new file mode 100644 index 00000000000..624fd00367e --- /dev/null +++ b/lib/gitlab/gl_id.rb @@ -0,0 +1,11 @@ +module Gitlab + module GlId + def self.gl_id(user) + if user.present? + "user-#{user.id}" + else + "" + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f751a3a12fd..d4f12cb1df9 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,7 +3,6 @@ module Gitlab def add_gon_variables gon.api_version = API::API.version gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_shortcuts_path diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 2122339d2db..3caf9036459 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= (@commits.size.to_f / @duration).round(1) + @commit_per_day ||= @commits.size / (@duration + 1) end def collect_data diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 280120b0f9e..41296415e35 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap). + def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false) + new(blob_name, blob_content, nowrap: nowrap, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -10,12 +10,21 @@ module Gitlab return [] unless blob blob.load_all_data!(repository) - highlight(file_name, blob.data).lines.map!(&:html_safe) + highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - def initialize(blob_name, blob_content, nowrap: true) + attr_reader :lexer + def initialize(blob_name, blob_content, repository: nil, nowrap: true) + @blob_name = blob_name + @blob_content = blob_content + @repository = repository @formatter = rouge_formatter(nowrap: nowrap) - @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + + @lexer = custom_language || begin + Rouge::Lexer.guess(filename: blob_name, source: blob_content).new + rescue Rouge::Lexer::AmbiguousGuess => e + e.alternatives.sort_by(&:tag).first + end end def highlight(text, continue: true, plain: false) @@ -30,6 +39,14 @@ module Gitlab private + def custom_language + language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') + + return nil unless language_name + + Rouge::Lexer.find_fancy(language_name) + end + def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb new file mode 100644 index 00000000000..588647e5adb --- /dev/null +++ b/lib/gitlab/import_export.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + extend self + + VERSION = '0.1.1' + + def export_path(relative_path:) + File.join(storage_path, relative_path) + end + + def storage_path + File.join(Settings.shared['path'], 'tmp/project_exports') + end + + def project_filename + "project.json" + end + + def project_bundle_filename + "project.bundle" + end + + def config_file + Rails.root.join('lib/gitlab/import_export/import_export.yml') + end + + def version_filename + 'VERSION' + end + + def version + VERSION + end + + def reset_tokens? + true + end + end +end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..d230de781d5 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,47 @@ +module Gitlab + module ImportExport + class AttributesFinder + + def initialize(included_attributes:, excluded_attributes:, methods:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + @methods = methods || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + + private + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..2249904145c --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,41 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def git_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) + end + + private + + def tar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir} .)) + end + + def untar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) + output, status = Gitlab::Popen.popen(cmd) + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status.zero? + status.zero? + end + + def git_bin_path + Gitlab.config.git.bin_path + end + end + end +end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..e341c4d9cf8 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..82d1e1805c5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,34 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + result = untar_zxf(archive: @archive_file, dir: @shared.export_path) + + raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + + true + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..05f4ad527ac --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,59 @@ +# Model relationships to be included in the project import/export +project_tree: + - issues: + - :events + - notes: + - :author + - :events + - :labels + - milestones: + - :events + - snippets: + - notes: + :author + - :releases + - project_members: + - :user + - merge_requests: + - notes: + - :author + - :events + - :merge_request_diff + - :events + - pipelines: + - notes: + - :author + - :events + - :statuses + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches + +# Only include the following attributes for the models specified. +included_attributes: + project: + - :description + - :issues_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + user: + - :id + - :email + - :username + author: + - :name + +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: + - :expired_at + +methods: + statuses: + - :type
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..8f66f48cbfe --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,72 @@ +module Gitlab + module ImportExport + class Importer + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + end + + def execute + if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end + + remove_import_file + end + + private + + def import_file + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + end + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..b459054c198 --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,67 @@ +module Gitlab + module ImportExport + class MembersMapper + attr_reader :missing_author_ids + + def initialize(exported_members:, user:, project:) + @exported_members = exported_members + @user = user + @project = project + @missing_author_ids = [] + + # This needs to run first, as second call would be from #map + # which means project members already exist. + ensure_default_member! + end + + def map + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def default_user_id + @user.id + end + + private + + def missing_keys_tracking_hash + Hash.new do |_, key| + @missing_author_ids << key + default_user_id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) + member['user'] = existing_user + + ProjectMember.create(member_hash(member)).persisted? + end + + def member_hash(member) + member.except('id').merge(source_id: @project.id, importing: true) + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..77bb3ca6581 --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,23 @@ +module Gitlab + module ImportExport + class ProjectCreator + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..025ecc12f9f --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,113 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + def initialize(user:, shared:, project:) + @path = File.join(shared.export_path, 'project.json') + @user = user + @shared = shared + @project = project + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + rescue => e + @shared.error(e) + false + end + + def restored_project + @restored_project ||= restore_project + end + + private + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project: restored_project) + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations + saved = [] + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + saved << restored_project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end + end + + def restore_project + return @project unless @tree_hash + + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + @project.update(project_params) + @project + end + + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash + # + # Recursively calls this method if the sub-relation is a hash containing more sub-relations + def create_sub_relations(relation, tree_hash) + relation_key = relation.keys.first.to_s + return if tree_hash[relation_key].blank? + + tree_hash[relation_key].each do |relation_item| + relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author + + create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + end + + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + [relation_hash, sub_relation] + end + + def create_relation(relation, relation_hash_list) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => restored_project.id), + members_mapper: members_mapper, + user: @user) + end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..9153088e966 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(full_path, project_json_tree) + true + rescue => e + @shared.error(e) + false + end + + private + + def project_json_tree + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..15f5dd31035 --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,115 @@ +module Gitlab + module ImportExport + class Reader + attr_reader :tree + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..9824df3f274 --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,130 @@ +module Gitlab + module ImportExport + class RelationFactory + OVERRIDES = { snippets: :project_snippets, + pipelines: 'Ci::Pipeline', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + + BUILD_MODELS = %w[Ci::Build commit_status].freeze + + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user = user + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] + + generate_imported_object + end + + private + + def update_user_references + USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + + # Users with admin access can map users + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id + + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + end + + def missing_author?(old_author_id) + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def generate_imported_object + if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil + end + else + imported_object + end + end + + def update_project_references + project_id = @relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end + end + + def reset_ci_tokens + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil + end + + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize + end + + def imported_object + imported_object = relation_class.new(@relation_hash) + yield(imported_object) if block_given? + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user.is_admin? + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..546dae4d122 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:, path_to_bundle:, wiki: false) + @project = project + @path_to_bundle = path_to_bundle + @shared = shared + @wiki = wiki + end + + def restore + return wiki? unless File.exist?(@path_to_bundle) + + FileUtils.mkdir_p(path_to_repo) + + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue => e + @shared.error(e) + false + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + + def wiki? + @wiki + end + end + end +end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..cce43fe994b --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..6a60b65071f --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,43 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + if compress_and_save + remove_export_path + Rails.logger.info("Saved project export #{archive_file}") + archive_file + else + @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) + false + end + rescue => e + @shared.error(e) + false + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @shared.export_path) + end + + def remove_export_path + FileUtils.rm_rf(@shared.export_path) + end + + def archive_file + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..5d6de8bc475 --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class Shared + attr_reader :errors, :opts + + def initialize(opts) + @opts = opts + @errors = [] + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + end + + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..df19354b76e --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,14 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + def restore + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..d6f4fa57510 --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class UploadsSaver + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + copy_files(uploads_path, uploads_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + end + end + end +end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..abfc694b879 --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class VersionChecker + def self.check!(*args) + new(*args).check! + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..9b642d740b7 --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class VersionSaver + def initialize(shared:) + @shared = shared + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..1eedae39f8a --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..59a05411fe9 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,11 +20,10 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', + 'GitLab export' => 'gitlab_project' } end - end - end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index 8684b4636ea..b75ae512d92 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -39,7 +39,7 @@ module Gitlab # OpenSSH 6.8 introduces a new default output format for fingerprints. # Check the version and decide which command to use. - version_output, version_status = popen(%W(ssh -V)) + version_output, version_status = popen(%w(ssh -V)) return false unless version_status.zero? version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 9d9617761b3..811363405a8 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -1,11 +1,11 @@ module Gitlab module Lfs class Response - - def initialize(project, user, request) + def initialize(project, user, ci, request) @origin_project = project @project = storage_project(project) @user = user + @ci = ci @env = request.env @request = request end @@ -189,7 +189,7 @@ module Gitlab return render_not_enabled unless Gitlab.config.lfs.enabled unless @project.public? - return render_unauthorized unless @user + return render_unauthorized unless @user || @ci return render_forbidden unless user_can_fetch? end @@ -210,7 +210,7 @@ module Gitlab def user_can_fetch? # Check user access against the project they used to initiate the pull - @user.can?(:download_code, @origin_project) + @ci || @user.can?(:download_code, @origin_project) end def user_can_push? diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 78d02891102..69bd5e62305 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -1,9 +1,12 @@ module Gitlab module Lfs class Router - def initialize(project, user, request) + attr_reader :project, :user, :ci, :request + + def initialize(project, user, ci, request) @project = project @user = user + @ci = ci @env = request.env @request = request end @@ -80,7 +83,7 @@ module Gitlab def lfs return unless @project - Gitlab::Lfs::Response.new(@project, @user, @request) + Gitlab::Lfs::Response.new(@project, @user, @ci, @request) end def sanitize_tmp_filename(name) diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d81d26754fe..dcec7543c13 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -148,23 +148,8 @@ module Gitlab proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) - trans = Gitlab::Metrics::Instrumentation.transaction - - if trans - start = Time.now - cpu_start = Gitlab::Metrics::System.cpu_time - retval = super - duration = (Time.now - start) * 1000.0 - - if duration >= Gitlab::Metrics.method_call_threshold - cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start - - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration, cpu_duration: cpu_duration }, - method: #{label.inspect}) - end - - retval + if trans = Gitlab::Metrics::Instrumentation.transaction + trans.measure_method(#{label.inspect}) { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb new file mode 100644 index 00000000000..c048fe20ba7 --- /dev/null +++ b/lib/gitlab/metrics/method_call.rb @@ -0,0 +1,52 @@ +module Gitlab + module Metrics + # Class for tracking timing information about method calls + class MethodCall + attr_reader :real_time, :cpu_time, :call_count + + # name - The full name of the method (including namespace) such as + # `User#sign_in`. + # + # series - The series to use for storing the data. + def initialize(name, series) + @name = name + @series = series + @real_time = 0.0 + @cpu_time = 0.0 + @call_count = 0 + end + + # Measures the real and CPU execution time of the supplied block. + def measure + start_real = System.monotonic_time + start_cpu = System.cpu_time + retval = yield + + @real_time += System.monotonic_time - start_real + @cpu_time += System.cpu_time - start_cpu + @call_count += 1 + + retval + end + + # Returns a Metric instance of the current method call. + def to_metric + Metric.new( + @series, + { + duration: real_time, + cpu_duration: cpu_time, + call_count: call_count + }, + method: @name + ) + end + + # Returns true if the total runtime of this method exceeds the method call + # threshold. + def above_threshold? + real_time >= Metrics.method_call_threshold + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 1cd1ca30f70..f23d67e1e38 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,16 +4,15 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags, :created_at + attr_reader :series, :values, :tags # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. def initialize(series, values, tags = {}) - @values = values - @series = series - @tags = tags - @created_at = Time.now.utc + @values = values + @series = series + @tags = tags end # Returns a Hash in a format that can be directly written to InfluxDB. @@ -27,20 +26,20 @@ module Gitlab # # Due to the way InfluxDB is set up there's no solution to this problem, # all we can do is lower the amount of collisions. We do this by using - # Time#to_f which returns the seconds as a Float providing greater - # accuracy. We then add a small random value that is large enough to - # distinguish most timestamps but small enough to not alter the amount - # of seconds. + # System.real_time which returns the nanoseconds as a Float providing + # greater accuracy. We then add a small random value that is large + # enough to distinguish most timestamps but small enough to not alter + # the timestamp significantly. # # See https://gitlab.com/gitlab-com/operations/issues/175 for more # information. - time = @created_at.to_f + rand(JITTER_RANGE) + time = System.real_time(:nanosecond) + rand(JITTER_RANGE) { series: @series, tags: @tags, values: @values, - timestamp: (time * 1_000_000_000).to_i + timestamp: time.to_i } end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 3fe27779d03..e61670f491c 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -35,7 +35,7 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, filtered_path(env)) trans.set(:request_method, env['REQUEST_METHOD']) trans @@ -54,6 +54,10 @@ module Gitlab private + def filtered_path(env) + ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] + end + def endpoint_paths_cache @endpoint_paths_cache ||= Hash.new do |hash, http_method| hash[http_method] = Hash.new do |inner_hash, raw_path| diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index fd98aa3412e..a1240fd33ee 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -8,6 +8,8 @@ module Gitlab trans = Transaction.new("#{worker.class.name}#perform") begin + # Old gitlad-shell messages don't provide enqueued_at/created_at attributes + trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } ensure trans.finish diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 8e345e8ae4a..aaed2184f44 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -2,11 +2,21 @@ module Gitlab module Metrics module Subscribers # Class for tracking the total time spent in Rails cache calls + # http://guides.rubyonrails.org/active_support_instrumentation.html class RailsCache < ActiveSupport::Subscriber attach_to :active_support def cache_read(event) increment(:cache_read, event.duration) + + return unless current_transaction + return if event.payload[:super_operation] == :fetch + + if event.payload[:hit] + current_transaction.increment(:cache_read_hit_count, 1) + else + current_transaction.increment(:cache_read_miss_count, 1) + end end def cache_write(event) @@ -21,6 +31,18 @@ module Gitlab increment(:cache_exists, event.duration) end + def cache_fetch_hit(event) + return unless current_transaction + + current_transaction.increment(:cache_read_hit_count, 1) + end + + def cache_generate(event) + return unless current_transaction + + current_transaction.increment(:cache_read_miss_count, 1) + end + def increment(key, duration) return unless current_transaction diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a7d183b2f94..82c18bb108b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,29 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f end else def self.cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f end end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float. + def self.real_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + end + + # Returns the current monotonic clock time in a given precision. + # + # Returns the time as a Float. + def self.monotonic_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 2578ddc49f4..bded245da43 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values + attr_reader :tags, :values, :methods attr_accessor :action @@ -16,6 +16,7 @@ module Gitlab # plus method name. def initialize(action = nil) @metrics = [] + @methods = {} @started_at = nil @finished_at = nil @@ -29,7 +30,7 @@ module Gitlab end def duration - @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + @finished_at ? (@finished_at - @started_at) : 0.0 end def allocated_memory @@ -40,20 +41,34 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = Time.now + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = Time.now + @finished_at = System.monotonic_time Thread.current[THREAD_KEY] = nil end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + end + + # Measures the time it takes to execute a method. + # + # Multiple calls to the same method add up to the total runtime of the + # method. + # + # name - The full name of the method to measure (e.g. `User#sign_in`). + def measure_method(name, &block) + unless @methods[name] + series = "#{series_prefix}#{Instrumentation::SERIES}" + + @methods[name] = MethodCall.new(name, series) + end - @metrics << Metric.new("#{prefix}#{series}", values, tags) + @methods[name].measure(&block) end def increment(name, value) @@ -84,7 +99,13 @@ module Gitlab end def submit - metrics = @metrics.map do |metric| + submit = @metrics.dup + + @methods.each do |name, method| + submit << method.to_metric if method.above_threshold? + end + + submit_hashes = submit.map do |metric| hash = metric.to_hash hash[:tags][:action] ||= @action if @action @@ -92,12 +113,16 @@ module Gitlab hash end - Metrics.submit_metrics(metrics) + Metrics.submit_metrics(submit_hashes) end def sidekiq? Sidekiq.server? end + + def series_prefix + sidekiq? ? 'sidekiq_' : 'rails_' + end end end end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 36e5c2670bb..7d6911a1ab3 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -66,7 +66,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 78f3ecb4cb4..0a91d3918d5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -56,8 +56,6 @@ module Gitlab if external_provider? && @user @user.external = true - elsif @user - @user.external = false end @user @@ -74,7 +72,7 @@ module Gitlab if user # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) + user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) else log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." user = find_by_uid_and_provider diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 746ec283330..4e2f8ed5587 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -1,7 +1,6 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup - # Public: Converts the provided markup into HTML. # # input - the source text in a markup format diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb new file mode 100644 index 00000000000..21aefc884be --- /dev/null +++ b/lib/gitlab/protocol_access.rb @@ -0,0 +1,13 @@ +module Gitlab + module ProtocolAccess + def self.allowed?(protocol) + if protocol == 'web' + true + elsif current_application_settings.enabled_git_access_protocol.blank? + true + else + protocol == current_application_settings.enabled_git_access_protocol + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1cbd6d945a0..ffad5e17c78 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -13,7 +13,6 @@ module Gitlab "Cannot start with '-' or end in '.'." \ end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -22,7 +21,6 @@ module Gitlab "can contain only letters, digits, '_', '.', dash and space." end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -32,7 +30,6 @@ module Gitlab "It must start with letter, digit or '_'." end - def project_path_regex @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end @@ -42,7 +39,6 @@ module Gitlab "Cannot start with '-', end in '.git' or end in '.atom'" \ end - def file_name_regex @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end @@ -59,7 +55,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end - def directory_traversal_regex @directory_traversal_regex ||= /\.{2}/.freeze end @@ -68,7 +63,6 @@ module Gitlab "cannot include directory traversal. " end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze @@ -100,5 +94,13 @@ module Gitlab def container_registry_reference_regex git_reference_regex end + + def environment_name_regex + @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + end + + def environment_name_regex_message + "can contain only letters, digits, '-' and '_'." + end end end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 32c1c9ec5bb..67a5f368bdb 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash - def groups get_raw(Gitlab::Saml::Config.groups) end @@ -13,7 +12,6 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end - end end end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb index 0f40c00f547..574c3a4b28c 100644 --- a/lib/gitlab/saml/config.rb +++ b/lib/gitlab/saml/config.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class Config - class << self def options Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } @@ -15,7 +14,6 @@ module Gitlab options[:external_groups] end end - end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8943022612c..f253dc7477e 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -6,7 +6,6 @@ module Gitlab module Saml class User < Gitlab::OAuth::User - def save super('SAML') end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index ae85b294d31..104280f520a 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,18 +25,18 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill('SIGTERM', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + "#{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb new file mode 100644 index 00000000000..760ff3e614a --- /dev/null +++ b/lib/gitlab/template/base_template.rb @@ -0,0 +1,67 @@ +module Gitlab + module Template + class BaseTemplate + def initialize(path) + @path = path + end + + def name + File.basename(@path, self.class.extension) + end + + def content + File.read(@path) + end + + class << self + def all + self.categories.keys.flat_map { |cat| by_category(cat) } + end + + def find(key) + file_name = "#{key}#{self.extension}" + + directory = select_directory(file_name) + directory ? new(File.join(category_directory(directory), file_name)) : nil + end + + def categories + raise NotImplementedError + end + + def extension + raise NotImplementedError + end + + def base_dir + raise NotImplementedError + end + + def by_category(category) + templates_for_directory(category_directory(category)) + end + + def category_directory(category) + File.join(base_dir, categories[category]) + end + + private + + def select_directory(file_name) + categories.keys.find do |category| + File.exist?(File.join(category_directory(category), file_name)) + end + end + + def templates_for_directory(dir) + dir << '/' unless dir.end_with?('/') + Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) } + end + + def filter_regex + @filter_reges ||= /#{Regexp.escape(extension)}\z/ + end + end + end + end +end diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore.rb new file mode 100644 index 00000000000..964fbfd4de3 --- /dev/null +++ b/lib/gitlab/template/gitignore.rb @@ -0,0 +1,22 @@ +module Gitlab + module Template + class Gitignore < BaseTemplate + class << self + def extension + '.gitignore' + end + + def categories + { + "Languages" => '', + "Global" => 'Global' + } + end + + def base_dir + Rails.root.join('vendor/gitignore') + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml.rb new file mode 100644 index 00000000000..7f480fe33c0 --- /dev/null +++ b/lib/gitlab/template/gitlab_ci_yml.rb @@ -0,0 +1,27 @@ +module Gitlab + module Template + class GitlabCiYml < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + '.gitlab-ci.yml' + end + + def categories + { + "General" => '', + "Pages" => 'Pages' + } + end + + def base_dir + Rails.root.join('vendor/gitlab-ci-yml') + end + end + end + end +end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb new file mode 100644 index 00000000000..b290c716f97 --- /dev/null +++ b/lib/gitlab/timeless.rb @@ -0,0 +1,16 @@ +module Gitlab + module Timeless + def self.timeless(model, &block) + original_record_timestamps = model.record_timestamps + model.record_timestamps = false + + if block.arity.abs == 1 + block.call(model) + else + block.call + end + ensure + model.record_timestamps = original_record_timestamps + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 7d02fe3c971..19dad699edf 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -4,10 +4,20 @@ module Gitlab regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) content.gsub(regexp) { |url| new(url).masked_url } + rescue Addressable::URI::InvalidURIError + content.gsub(regexp, '') + end + + def self.valid?(url) + Addressable::URI.parse(url.strip) + + true + rescue Addressable::URI::InvalidURIError + false end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url) + @url = Addressable::URI.parse(url.strip) @credentials = credentials end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 388f84dbe0e..6aeb49c0219 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -8,7 +8,7 @@ module Gitlab class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::ShellEnv.gl_id(user), + 'GL_ID' => Gitlab::GlId.gl_id(user), 'RepoPath' => repository.path_to_repo, } end @@ -38,12 +38,10 @@ module Gitlab end def send_git_diff(repository, diff_refs) - from, to = diff_refs - params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => from.sha, - 'ShaTo' => to.sha + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha } [ @@ -52,6 +50,31 @@ module Gitlab ] end + def send_git_patch(repository, diff_refs) + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha + } + + [ + SEND_DATA_HEADER, + "git-format-patch:#{encode(params)}" + ] + end + + def send_artifacts_entry(build, entry) + params = { + 'Archive' => build.artifacts_file.path, + 'Entry' => Base64.encode64(entry.path) + } + + [ + SEND_DATA_HEADER, + "artifacts-entry:#{encode(params)}" + ] + end + protected def encode(hash) diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 8c309efc7b8..3358ed6773e 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -143,18 +143,14 @@ module Rouge '</span>' end end - lines.join("\n") - else - if @linenos == 'inline' - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - "<span class=\"linenos\">#{number}</span>#{line}" - end - lines.join("\n") - else - lines.join("\n") + elsif @linenos == 'inline' + lines = lines.each_with_index.map do |line, index| + number = index + @linenostart + "<span class=\"linenos\">#{number}</span>#{line}" end end + + lines.join("\n") end def span(tok, val) diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..4a4892a2e07 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -49,7 +49,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..0b93d7f292f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -93,7 +93,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 030ee8bafcb..e930ace1041 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -13,7 +13,7 @@ namespace :gemojione do aliases[real_name] << alias_name end - AwardEmoji.emojis.map do |name, emoji_hash| + Gitlab::AwardEmoji.emojis.map do |name, emoji_hash| fpath = File.join(dir, "#{emoji_hash['unicode']}.png") digest = Digest::SHA256.file(fpath).hexdigest diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9ee72fde92f..b43ee5b3383 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,12 +33,13 @@ namespace :gitlab do unless backup.skipped?('db') unless ENV['force'] == 'yes' - warning = warning = <<-MSG.strip_heredoc + warning = <<-MSG.strip_heredoc Before restoring the database we recommend removing all existing tables to avoid future upgrade problems. Be aware that if you have custom tables in the GitLab database these tables and all data will be removed. MSG + puts warning.color(:red) ask_to_continue puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake new file mode 100644 index 00000000000..c2c6031db67 --- /dev/null +++ b/lib/tasks/gitlab/import_export.rake @@ -0,0 +1,13 @@ +namespace :gitlab do + namespace :import_export do + desc "GitLab | Show Import/Export version" + task version: :environment do + puts "Import/Export v#{Gitlab::ImportExport.version}" + end + + desc "GitLab | Display exported DB structure" + task data: :environment do + puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true) + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake deleted file mode 100644 index 4fd48cccb1d..00000000000 --- a/lib/tasks/gitlab/update_gitignore.rake +++ /dev/null @@ -1,46 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update gitignore" - task :update_gitignore do - unless clone_gitignores - puts "Cloning the gitignores failed".color(:red) - return - end - - remove_unneeded_files(gitignore_directory) - remove_unneeded_files(global_directory) - - puts "Done".color(:green) - end - - def clone_gitignores - FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) - FileUtils.cd vendor_directory - - system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') - end - - # Retain only certain files: - # - The LICENSE, because we have to - # - The sub dir global - # - The gitignores themself - # - Dir.entires returns also the entries '.' and '..' - def remove_unneeded_files(path) - Dir.foreach(path) do |file| - FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ - end - end - - private - - def vendor_directory - Rails.root.join('vendor') - end - - def gitignore_directory - File.join(vendor_directory, 'gitignore') - end - - def global_directory - File.join(gitignore_directory, 'Global') - end -end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake new file mode 100644 index 00000000000..4f76dad7286 --- /dev/null +++ b/lib/tasks/gitlab/update_templates.rake @@ -0,0 +1,54 @@ +namespace :gitlab do + desc "GitLab | Update templates" + task :update_templates do + TEMPLATE_DATA.each { |template| update(template) } + end + + def update(template) + sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + dir = File.join(vendor_directory, sub_dir) + + unless clone_repository(template.repo_url, dir) + puts "Cloning the #{sub_dir} templates failed".red + return + end + + remove_unneeded_files(dir, template.cleanup_regex) + puts "Done".green + end + + def clone_repository(url, directory) + FileUtils.rm_rf(directory) if Dir.exist?(directory) + + system("git clone #{url} --depth=1 --branch=master #{directory}") + end + + # Retain only certain files: + # - The LICENSE, because we have to + # - The sub dirs so we can organise the file by category + # - The templates themself + # - Dir.entries returns also the entries '.' and '..' + def remove_unneeded_files(directory, regex) + Dir.foreach(directory) do |file| + FileUtils.rm_rf(File.join(directory, file)) unless file =~ regex + end + end + + private + + Template = Struct.new(:repo_url, :cleanup_regex) + TEMPLATE_DATA = [ + Template.new( + "https://github.com/github/gitignore.git", + /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ + ), + Template.new( + "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", + /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/ + ) + ] + + def vendor_directory + Rails.root.join('vendor') + end +end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index c5666d49e61..21c0e5f1d41 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -6,8 +6,6 @@ task :test do end unless Rails.env.production? - require 'coveralls/rake/task' - Coveralls::RakeTask.new desc "GitLab | Run all tests on CI with simplecov" - task :test_ci => [:rubocop, :brakeman, 'teaspoon', :spinach, :spec, 'coveralls:push'] + task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec] end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index d4291f012d3..41dee5fdc06 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -3,7 +3,6 @@ require "fileutils" # Taken from: Rack::Test::UploadedFile class UploadedFile - # The filename, *not* including the path, of the "uploaded" file attr_reader :original_filename |