diff options
Diffstat (limited to 'lib')
246 files changed, 6856 insertions, 1858 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 7d65145176b..0e7a1cc2623 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,5 +1,3 @@ -Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} - module API class API < Grape::API include APIGuard @@ -25,37 +23,43 @@ module API format :json content_type :txt, "text/plain" - helpers Helpers + # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::API::Helpers - mount Groups - mount GroupMembers - mount Users - mount Projects - mount Repositories - mount Issues - mount Milestones - mount Session - mount MergeRequests - mount Notes - mount Internal - mount SystemHooks - mount ProjectSnippets - mount ProjectMembers - mount DeployKeys - mount ProjectHooks - mount Services - mount Files - mount Commits - mount CommitStatus - mount Namespaces - mount Branches - mount Labels - mount Settings - mount Keys - mount Tags - mount Triggers - mount Builds - mount Variables - mount Runners + mount ::API::AwardEmoji + mount ::API::Branches + mount ::API::Builds + mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::DeployKeys + mount ::API::Files + mount ::API::Gitignores + mount ::API::GroupMembers + mount ::API::Groups + mount ::API::Internal + mount ::API::Issues + mount ::API::Keys + mount ::API::Labels + mount ::API::Licenses + mount ::API::MergeRequests + mount ::API::Milestones + mount ::API::Namespaces + mount ::API::Notes + mount ::API::ProjectHooks + mount ::API::ProjectMembers + mount ::API::ProjectSnippets + mount ::API::Projects + mount ::API::Repositories + mount ::API::Runners + mount ::API::Services + mount ::API::Session + mount ::API::Settings + mount ::API::SidekiqMetrics + mount ::API::Subscriptions + mount ::API::SystemHooks + mount ::API::Tags + mount ::API::Triggers + mount ::API::Users + mount ::API::Variables end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9994fcefda..7e67edb203a 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -2,171 +2,175 @@ require 'rack/oauth2' -module APIGuard - extend ActiveSupport::Concern +module API + module APIGuard + extend ActiveSupport::Concern - included do |base| - # OAuth2 Resource Server Authentication - use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| - # The authenticator only fetches the raw token string + included do |base| + # OAuth2 Resource Server Authentication + use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| + # The authenticator only fetches the raw token string - # Must yield access token to store it in the env - request.access_token - end + # Must yield access token to store it in the env + request.access_token + end - helpers HelperMethods + helpers HelperMethods - install_error_responders(base) - end + install_error_responders(base) + end - # Helper Methods for Grape Endpoint - module HelperMethods - # Invokes the doorkeeper guard. - # - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it raises TokenNotFoundError. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + # Helper Methods for Grape Endpoint + module HelperMethods + # Invokes the doorkeeper guard. + # + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it raises TokenNotFoundError. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def doorkeeper_guard!(scopes: []) + if (access_token = find_access_token).nil? + raise TokenNotFoundError + + else + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + def doorkeeper_guard(scopes: []) + if access_token = find_access_token + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def current_user - @current_user - end + def current_user + @current_user + end - private - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) - end + private - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end + def find_access_token + @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + end - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end - end + def doorkeeper_request + @doorkeeper_request ||= ActionDispatch::Request.new(env) + end - module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes + def validate_access_token(access_token, scopes) + Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) end end - private - def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + module ClassMethods + # Installs the doorkeeper guard on the whole Grape API endpoint. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def guard_all!(scopes: []) + before do + guard! scopes: scopes + end + end - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler - end + private - def oauth2_bearer_token_error_handler - Proc.new do |e| - response = - case e - when MissingTokenError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - - when TokenNotFoundError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Bad Access Token.") - - when ExpiredError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token is expired. You can either do re-authorization or token refresh.") - - when RevokedError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token was revoked. You have to re-authorize from the user.") - - when InsufficientScopeError - # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) - # does not include WWW-Authenticate header, which breaks the standard. - Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( - :insufficient_scope, - Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], - { scope: e.scopes }) - end + def install_error_responders(base) + error_classes = [ MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] - response.finish + base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + end + + def oauth2_bearer_token_error_handler + Proc.new do |e| + response = + case e + when MissingTokenError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new + + when TokenNotFoundError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Bad Access Token.") + + when ExpiredError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is expired. You can either do re-authorization or token refresh.") + + when RevokedError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token was revoked. You have to re-authorize from the user.") + + when InsufficientScopeError + # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) + # does not include WWW-Authenticate header, which breaks the standard. + Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( + :insufficient_scope, + Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], + { scope: e.scopes }) + end + + response.finish + end end end - end - # - # Exceptions - # + # + # Exceptions + # - class MissingTokenError < StandardError; end + class MissingTokenError < StandardError; end - class TokenNotFoundError < StandardError; end + class TokenNotFoundError < StandardError; end - class ExpiredError < StandardError; end + class ExpiredError < StandardError; end - class RevokedError < StandardError; end + class RevokedError < StandardError; end - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes + class InsufficientScopeError < StandardError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes + end end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb new file mode 100644 index 00000000000..985590312e3 --- /dev/null +++ b/lib/api/award_emoji.rb @@ -0,0 +1,116 @@ +module API + class AwardEmoji < Grape::API + before { authenticate! } + AWARDABLES = [Issue, MergeRequest] + + resource :projects do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.to_s.underscore.pluralize + awardable_id_string = "#{awardable_type.to_s.underscore}_id" + + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get endpoint do + if can_read_awardable? + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or mr + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:awardable_id/award_emoji + post endpoint do + required_attributes! [:name] + + not_found!('Award Emoji') unless can_read_awardable? + + award = awardable.award_emoji.new(name: params[:name], user: current_user) + + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + end + + helpers do + def can_read_awardable? + ability = "read_#{awardable.class.to_s.underscore}".to_sym + + can?(current_user, ability, awardable) + end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + noteable.notes.find(params[:note_id]) + else + noteable + end + end + end + + def noteable + if params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + else + user_project.merge_requests.find(params[:merge_request_id]) + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 592100a7045..231840148d9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -64,7 +64,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch does not exist") unless @branch + not_found!("Branch") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 2b104f90aa7..979328efe0e 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -33,7 +33,7 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.ci_commits.find_by_sha(params[:sha]) + commit = user_project.pipelines.find_by_sha(params[:sha]) return not_found! unless commit builds = commit.builds.order('id DESC') @@ -142,7 +142,7 @@ module API return not_found!(build) unless build return forbidden!('Build is not retryable') unless build.retryable? - build = Ci::Build.retry(build) + build = Ci::Build.retry(build, current_user) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) @@ -166,6 +166,26 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end + + # Keep the artifacts to prevent them from being deleted + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - The ID of a build + # Example Request: + # POST /projects/:id/builds/:build_id/artifacts/keep + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 8e74e177ea0..323a7086890 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -2,7 +2,7 @@ require 'mime/types' module API # Project commit statuses API - class CommitStatus < Grape::API + class CommitStatuses < Grape::API resource :projects do before { authenticate! } @@ -21,10 +21,9 @@ module API authorize!(:read_commit_status, user_project) not_found!('Commit') unless user_project.commit(params[:sha]) - ci_commit = user_project.ci_commit(params[:sha]) - return [] unless ci_commit - statuses = ci_commit.statuses + pipelines = user_project.pipelines.where(sha: params[:sha]) + statuses = ::CommitStatus.where(pipeline: pipelines) statuses = statuses.latest unless parse_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? statuses = statuses.where(stage: params[:stage]) if params[:stage].present? @@ -51,11 +50,25 @@ module API commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit - ci_commit = @project.ensure_ci_commit(commit.sha) + # Since the CommitStatus is attached to Ci::Pipeline (in the future Pipeline) + # We need to always have the pipeline object + # To have a valid pipeline object that can be attached to specific MR + # Other CI service needs to send `ref` + # If we don't receive it, we will attach the CommitStatus to + # the first found branch on that commit + + ref = params[:ref] + unless ref + branches = @project.repository.branch_names_contains(commit.sha) + not_found! 'References for commit' if branches.none? + ref = branches.first + end + + pipeline = @project.ensure_pipeline(commit.sha, ref) name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, commit: ci_commit, user: current_user) + status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) + status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) status.update(attrs) case params[:state].to_s diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4544a41b1e3..4a11c8e3620 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -12,14 +12,20 @@ module API # Parameters: # id (required) - The ID of a project # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used + # since (optional) - Only commits after or in this date will be returned + # until (optional) - Only commits before or in this date will be returned # Example Request: # GET /projects/:id/repository/commits get ":id/repository/commits" do + datetime_attributes! :since, :until + page = (params[:page] || 0).to_i per_page = (params[:per_page] || 20).to_i ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + after = params[:since] + before = params[:until] - commits = user_project.repository.commits(ref, nil, per_page, page * per_page) + commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before) present commits, with: Entities::RepoCommit end @@ -101,6 +107,8 @@ module API break if opts[:line_code] end + + opts[:type] = LegacyDiffNote.name if opts[:line_code] end note = ::Notes::CreateService.new(user_project, current_user, opts).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 71197205f34..2e397643ed1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -8,14 +8,14 @@ module API expose :id, :state, :avatar_url expose :web_url do |user, options| - Gitlab::Application.routes.url_helpers.user_url(user) + Gitlab::Routing.url_helpers.user_url(user) end end class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url end class Identity < Grape::Entity @@ -30,7 +30,7 @@ module API expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project - expose :two_factor_enabled + expose :two_factor_enabled?, as: :two_factor_enabled expose :external end @@ -66,7 +66,8 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :created_at, :last_activity_at expose :shared_runners_enabled expose :creator_id expose :namespace @@ -85,12 +86,9 @@ module API end class Group < Grape::Entity - expose :id, :name, :path, :description + expose :id, :name, :path, :description, :visibility_level expose :avatar_url - - expose :web_url do |group, options| - Gitlab::Application.routes.url_helpers.group_url(group) - end + expose :web_url end class GroupDetail < Group @@ -170,11 +168,22 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic + + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user]) + end + expose :user_notes_count + expose :upvotes, :downvotes + end + + class ExternalIssue < Grape::Entity + expose :title + expose :id end class MergeRequest < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels @@ -183,6 +192,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user]) + end + expose :user_notes_count end class MergeRequestChanges < MergeRequest @@ -204,12 +217,20 @@ module API expose :note, as: :body expose :attachment_identifier, as: :attachment expose :author, using: Entities::UserBasic - expose :created_at + expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type # upvote? and downvote? are deprecated, always return false - expose :upvote?, as: :upvote - expose :downvote?, as: :downvote + expose(:upvote?) { |note| false } + expose(:downvote?) { |note| false } + end + + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type end class MRNote < Grape::Entity @@ -219,9 +240,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_name } - expose(:line) { |note| note.diff_new_line } - expose(:line_type) { |note| note.diff_line_type } + expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } + expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } + expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } expose :author, using: Entities::UserBasic expose :created_at end @@ -255,14 +276,19 @@ module API expose :id, :path, :kind end - class ProjectAccess < Grape::Entity + class Member < Grape::Entity expose :access_level - expose :notification_level + expose :notification_level do |member, options| + if member.notification_setting + NotificationSetting.levels[member.notification_setting.level] + end + end end - class GroupAccess < Grape::Entity - expose :access_level - expose :notification_level + class ProjectAccess < Member + end + + class GroupAccess < Member end class ProjectService < Grape::Entity @@ -292,7 +318,12 @@ module API end class Label < Grape::Entity - expose :name, :color + expose :name, :color, :description + expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + + expose :subscribed do |label, options| + label.subscribed?(options[:current_user]) + end end class Compare < Grape::Entity @@ -330,19 +361,21 @@ module API expose :signin_enabled expose :gravatar_enabled expose :sign_in_text + expose :after_sign_up_text expose :created_at expose :updated_at expose :home_page_url expose :default_branch_protection - expose :twitter_sharing_enabled expose :restricted_visibility_levels expose :max_attachment_size expose :session_expire_delay expose :default_project_visibility expose :default_snippet_visibility + expose :default_group_visibility expose :restricted_signup_domains expose :user_oauth_applications expose :after_sign_out_path + expose :container_registry_token_expire_delay end class Release < Grape::Entity @@ -389,6 +422,7 @@ module API class RunnerDetails < Runner expose :tag_list + expose :run_untagged expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -425,5 +459,25 @@ module API class Variable < Grape::Entity expose :key, :value end + + class RepoLicense < Grape::Entity + expose :key, :name, :nickname + expose :featured, as: :popular + expose :url, as: :html_url + expose(:source_url) { |license| license.meta['source'] } + expose(:description) { |license| license.meta['description'] } + expose(:conditions) { |license| license.meta['conditions'] } + expose(:permissions) { |license| license.meta['permissions'] } + expose(:limitations) { |license| license.meta['limitations'] } + expose :content + end + + class GitignoresList < Grape::Entity + expose :name + end + + class Gitignore < Grape::Entity + expose :name, :content + end end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb new file mode 100644 index 00000000000..270c9501dd2 --- /dev/null +++ b/lib/api/gitignores.rb @@ -0,0 +1,29 @@ +module API + class Gitignores < Grape::API + + # Get the list of the available gitignore templates + # + # Example Request: + # GET /gitignores + get 'gitignores' do + present Gitlab::Gitignore.all, with: Entities::GitignoresList + end + + # Get the text for a specific gitignore + # + # Parameters: + # name (required) - The name of a license + # + # Example Request: + # GET /gitignores/Elixir + # + get 'gitignores/:name' do + required_attributes! [:name] + + gitignore = Gitlab::Gitignore.find(params[:name]) + not_found!('.gitignore') unless gitignore + + present gitignore, with: Entities::Gitignore + end + end +end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 1a14d870a4a..9d8b8d737a9 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,15 +23,17 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group # Example Request: # POST /groups post do authorize! :create_group, current_user required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level] @group = Group.new(attrs) if @group.save @@ -42,6 +44,28 @@ module API end end + # Update group. Available only for users who can administrate groups. + # + # Parameters: + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # Example Request: + # PUT /groups/:id + put ':id' do + group = find_group(params[:id]) + authorize! :admin_group, group + + attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + + if ::Groups::UpdateService.new(group, current_user, attrs).execute + present group, with: Entities::GroupDetail + else + render_validation_error!(group) + end + end + # Get a single group, with containing projects # # Parameters: @@ -71,8 +95,7 @@ module API # GET /groups/:id/projects get ":id/projects" do group = find_group(params[:id]) - projects = group.projects - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects present projects, with: Entities::Project end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a72044e8058..77e407b54c5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,16 +2,20 @@ module API module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER ="HTTP_SUDO" + SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo def parse_boolean(value) [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end + def find_user_by_private_token + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + end + def current_user - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + @current_user ||= (find_user_by_private_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil @@ -29,11 +33,11 @@ module API @current_user end - def sudo_identifier() + def sudo_identifier identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers - if !!(identifier =~ /^[0-9]+$/) + if !!(identifier =~ /\A[0-9]+\z/) identifier.to_i else identifier @@ -91,11 +95,21 @@ module API if can?(current_user, :read_group, group) group else - forbidden!("#{current_user.username} lacks sufficient "\ - "access to #{group.name}") + not_found!('Group') end end + def find_project_label(id) + label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label || not_found!('Label') + end + + def find_project_issue(id) + issue = user_project.issues.find(id) + not_found! unless can?(current_user, :read_issue, issue) + issue + end + def paginate(relation) relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| add_pagination_headers(data) @@ -118,9 +132,7 @@ module API end def authorize!(action, subject) - unless abilities.allowed?(current_user, action, subject) - forbidden! - end + forbidden! unless abilities.allowed?(current_user, action, subject) end def authorize_push_project @@ -186,6 +198,22 @@ module API Gitlab::Access.options_with_owner.values.include? level.to_i end + # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 + # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. + # + # Parameters: + # keys (required) - An array consisting of elements that must be parseable as dates from the params hash + def datetime_attributes!(*keys) + keys.each do |key| + begin + params[key] = Time.xmlschema(params[key]) if params[key].present? + rescue ArgumentError + message = "\"" + key.to_s + "\" must be a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ" + render_api_error!(message, 400) + end + end + end + def issuable_order_by if params["order_by"] == 'updated_at' 'updated_at' @@ -243,6 +271,10 @@ module API render_api_error!('413 Request Entity Too Large', 413) end + def not_modified! + render_api_error!('304 Not Modified', 304) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -380,5 +412,23 @@ module API error!(errors[:access_level], 422) if errors[:access_level].any? not_found!(errors) end + + def send_git_blob(repository, blob) + env['api.format'] = :txt + content_type 'text/plain' + header(*Gitlab::Workhorse.send_git_blob(repository, blob)) + end + + def send_git_archive(repository, ref:, format:) + header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) + end + + def issue_entity(project) + if project.has_external_issue_tracker? + Entities::ExternalIssue + else + Entities::Issue + end + end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 2200208b946..3ac7b50c4ce 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -23,9 +23,11 @@ module API end post "/allowed" do + Gitlab::Metrics.action = 'Grape#/internal/allowed' + status 200 - actor = + actor = if params[:key_id] Key.find_by(id: params[:key_id]) elsif params[:user_id] @@ -33,7 +35,7 @@ module API end project_path = params[:project] - + # Check for *.wiki repositories. # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 252744515da..4c43257c48a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -24,8 +24,8 @@ module API def create_spam_log(project, current_user, attrs) params = attrs.merge({ - source_ip: env['REMOTE_ADDR'], - user_agent: env['HTTP_USER_AGENT'], + source_ip: client_ip(env), + user_agent: user_agent(env), noteable_type: 'Issue', via_api: true }) @@ -51,11 +51,11 @@ module API # GET /issues?labels=foo,bar # GET /issues?labels=foo,bar&state=opened get do - issues = current_user.issues + issues = current_user.issues.inc_notes_with_associations issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -82,7 +82,7 @@ module API # GET /projects/:id/issues?milestone=1.0.0&state=closed # GET /issues?iid=42 get ":id/issues" do - issues = user_project.issues + issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user) issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? @@ -92,7 +92,7 @@ module API end issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end # Get a single project issue @@ -103,24 +103,28 @@ module API # Example Request: # GET /projects/:id/issues/:issue_id get ":id/issues/:issue_id" do - @issue = user_project.issues.find(params[:issue_id]) - present @issue, with: Entities::Issue + @issue = find_project_issue(params[:issue_id]) + present @issue, with: Entities::Issue, current_user: current_user end # Create a new project issue # # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue + # id (required) - The ID of a project + # title (required) - The title of an issue + # description (optional) - The description of an issue + # assignee_id (optional) - The ID of a user to assign issue # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue + # labels (optional) - The labels of an issue + # created_at (optional) - Date time string, ISO 8601 formatted # Example Request: # POST /projects/:id/issues post ":id/issues" do required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id] + + keys = [:title, :description, :assignee_id, :milestone_id] + keys << :created_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -144,7 +148,7 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end @@ -161,12 +165,15 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) + # updated_at (optional) - Date time string, ISO 8601 formatted # Example Request: # PUT /projects/:id/issues/:issue_id put ":id/issues/:issue_id" do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys << :updated_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -184,13 +191,36 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end end - # Delete a project issue (deprecated) + # Move an existing issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # to_project_id (required) - The ID of the new project + # Example Request: + # POST /projects/:id/issues/:issue_id/move + post ':id/issues/:issue_id/move' do + required_attributes! [:to_project_id] + + issue = user_project.issues.find(params[:issue_id]) + new_project = Project.find(params[:to_project_id]) + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: Entities::Issue, current_user: current_user + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + # + # Delete a project issue # # Parameters: # id (required) - The ID of a project @@ -198,7 +228,10 @@ module API # Example Request: # DELETE /projects/:id/issues/:issue_id delete ":id/issues/:issue_id" do - not_allowed! + issue = user_project.issues.find_by(id: params[:issue_id]) + + authorize!(:destroy_issue, issue) + issue.destroy end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 78ca58ad0d1..c806829d69e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -11,23 +11,24 @@ module API # Example Request: # GET /projects/:id/labels get ':id/labels' do - present user_project.labels, with: Entities::Label + present user_project.labels, with: Entities::Label, current_user: current_user end # Creates a new label # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be created + # color (required) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # POST /projects/:id/labels post ':id/labels' do authorize! :admin_label, user_project required_attributes! [:name, :color] - attrs = attributes_for_keys [:name, :color] + attrs = attributes_for_keys [:name, :color, :description] label = user_project.find_label(attrs[:name]) conflict!('Label already exists') if label @@ -35,7 +36,7 @@ module API label = user_project.labels.create(attrs) if label.valid? - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end @@ -62,11 +63,12 @@ module API # Updates an existing label. At least one optional parameter is required. # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be deleted + # new_name (optional) - The new name of the label + # color (optional) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # PUT /projects/:id/labels put ':id/labels' do @@ -76,7 +78,7 @@ module API label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color] + attrs = attributes_for_keys [:new_name, :color, :description] if attrs.empty? render_api_error!('Required parameters "new_name" or "color" ' \ @@ -88,7 +90,7 @@ module API attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) if label.update(attrs) - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb new file mode 100644 index 00000000000..be0e113fbcb --- /dev/null +++ b/lib/api/licenses.rb @@ -0,0 +1,58 @@ +module API + # Licenses API + class Licenses < Grape::API + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + + # Get the list of the available license templates + # + # Parameters: + # popular - Filter licenses to only the popular ones + # + # Example Request: + # GET /licenses + # GET /licenses?popular=1 + get 'licenses' do + options = { + featured: params[:popular].present? ? true : nil + } + present Licensee::License.all(options), with: Entities::RepoLicense + end + + # Get text for specific license + # + # Parameters: + # key (required) - The key of a license + # project - Copyrighted project name + # fullname - Full name of copyright holder + # + # Example Request: + # GET /licenses/mit + # + get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do + required_attributes! [:key] + + not_found!('License') unless Licensee::License.find(params[:key]) + + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + license = Licensee::License.new(params[:key]) + + license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + + present license, with: Entities::RepoLicense + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c5e5d57ed4d..0e94efd4acd 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -41,7 +41,7 @@ module API # get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests + merge_requests = user_project.merge_requests.inc_notes_with_associations unless params[:iid].nil? merge_requests = filter_by_iid(merge_requests, params[:iid]) @@ -56,7 +56,7 @@ module API end merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user end # Create MR @@ -94,12 +94,24 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end end + # Delete a MR + # + # Parameters: + # id (required) - The ID of the project + # merge_request_id (required) - The MR id + delete ":id/merge_requests/:merge_request_id" do + merge_request = user_project.merge_requests.find_by(id: params[:merge_request_id]) + + authorize!(:destroy_merge_request, merge_request) + merge_request.destroy + end + # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 # Use "merge_requests/:merge_request_id/..." instead. # @@ -118,7 +130,7 @@ module API authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Show MR commits @@ -150,7 +162,7 @@ module API merge_request = user_project.merge_requests. find(params[:merge_request_id]) authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end # Update MR @@ -192,7 +204,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -206,6 +218,7 @@ module API # merge_commit_message (optional) - Custom merge commit message # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds + # sha (optional) - When present, must have the HEAD SHA of the source branch # Example: # PUT /projects/:id/merge_requests/:merge_request_id/merge # @@ -215,18 +228,21 @@ module API # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged + not_allowed! unless merge_request.mergeable_state? + + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? - render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? + if params[:sha] && merge_request.source_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.source_sha}", 409) + end merge_params = { commit_message: params[:merge_commit_message], should_remove_source_branch: params[:should_remove_source_branch] } - if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? + if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). execute(merge_request) else @@ -234,7 +250,7 @@ module API execute(merge_request) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Cancel Merge if Merge When build succeeds is enabled @@ -313,7 +329,7 @@ module API get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: issue_entity(user_project), current_user: current_user end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index c5cd73943fb..132043cf3f7 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -3,17 +3,35 @@ module API class Milestones < Grape::API before { authenticate! } + helpers do + def filter_milestones_state(milestones, state) + case state + when 'active' then milestones.active + when 'closed' then milestones.closed + else milestones + end + end + end + resource :projects do # Get a list of project milestones # # Parameters: - # id (required) - The ID of a project + # id (required) - The ID of a project + # state (optional) - Return "active" or "closed" milestones # Example Request: # GET /projects/:id/milestones + # GET /projects/:id/milestones?iid=42 + # GET /projects/:id/milestones?state=active + # GET /projects/:id/milestones?state=closed get ":id/milestones" do authorize! :read_milestone, user_project - present paginate(user_project.milestones), with: Entities::Milestone + milestones = user_project.milestones + milestones = filter_milestones_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + + present paginate(milestones), with: Entities::Milestone end # Get a single project milestone @@ -87,7 +105,15 @@ module API authorize! :read_milestone, user_project @milestone = user_project.milestones.find(params[:milestone_id]) - present paginate(@milestone.issues), with: Entities::Issue + + finder_params = { + project_id: user_project.id, + milestone_title: @milestone.title, + state: 'all' + } + + issues = IssuesFinder.new(current_user, finder_params).execute + present paginate(issues), with: Entities::Issue, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..8bfa998dc53 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -19,20 +19,24 @@ module API # GET /projects/:id/issues/:noteable_id/notes # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) + + if can?(current_user, noteable_read_ability_name(@noteable), @noteable) + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note + else + not_found!("Notes") + end end # Get a single +noteable+ note @@ -45,13 +49,14 @@ module API # GET /projects/:id/issues/:noteable_id/notes/:note_id # GET /projects/:id/snippets/:noteable_id/notes/:note_id get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) @note = @noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) - if @note.cross_reference_not_visible_for?(current_user) - not_found!("Note") - else + if can_read_note present @note, with: Entities::Note + else + not_found!("Note") end end @@ -61,6 +66,7 @@ module API # id (required) - The ID of a project # noteable_id (required) - The ID of an issue or snippet # body (required) - The content of a note + # created_at (optional) - The date # Example Request: # POST /projects/:id/issues/:noteable_id/notes # POST /projects/:id/snippets/:noteable_id/notes @@ -73,6 +79,10 @@ module API noteable_id: params[noteable_id_str] } + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end + @note = ::Notes::CreateService.new(user_project, current_user, opts).execute if @note.valid? @@ -112,6 +122,29 @@ module API end end + # Delete a +noteable+ note + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue, MR, or snippet + # node_id (required) - The ID of a note + # Example Request: + # DELETE /projects/:id/issues/:noteable_id/notes/:note_id + # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id + delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DeleteService.new(user_project, current_user).execute(note) + + present note, with: Entities::Note + end + end + end + + helpers do + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore}".to_sym end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index cf9938d25a7..ccca65cbe1c 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -103,10 +103,10 @@ module API required_attributes! [:hook_id] begin - @hook = ProjectHook.find(params[:hook_id]) - @hook.destroy + @hook = user_project.hooks.destroy(params[:hook_id]) rescue # ProjectHook can raise Error if hook_id not found + not_found!("Error deleting hook #{params[:hook_id]}") end end end diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index c756bb479fc..b703da0557a 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -46,7 +46,7 @@ module API required_attributes! [:user_id, :access_level] # either the user is already a team member or a new one - project_member = user_project.project_member_by_id(params[:user_id]) + project_member = user_project.project_member(params[:user_id]) if project_member.nil? project_member = user_project.project_members.new( user_id: params[:user_id], @@ -93,12 +93,17 @@ module API # Example Request: # DELETE /projects/:id/members/:user_id delete ":id/members/:user_id" do - authorize! :admin_project, user_project project_member = user_project.project_members.find_by(user_id: params[:user_id]) - unless project_member.nil? - project_member.destroy - else + + unless current_user.can?(:admin_project, user_project) || + current_user.can?(:destroy_project_member, project_member) + forbidden! + end + + if project_member.nil? { message: "Access revoked", id: params[:user_id].to_i } + else + project_member.destroy end end end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 22ce3c6a066..ce1bf0d26d2 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -11,6 +11,11 @@ module API end not_found! end + + def snippets_for_current_user + finder_params = { filter: :by_project, project: user_project } + SnippetsFinder.new.execute(current_user, finder_params) + end end # Get a project snippets @@ -20,7 +25,7 @@ module API # Example Request: # GET /projects/:id/snippets get ":id/snippets" do - present paginate(user_project.snippets), with: Entities::ProjectSnippet + present paginate(snippets_for_current_user), with: Entities::ProjectSnippet end # Get a project snippet @@ -31,7 +36,7 @@ module API # Example Request: # GET /projects/:id/snippets/:snippet_id get ":id/snippets/:snippet_id" do - @snippet = user_project.snippets.find(params[:snippet_id]) + @snippet = snippets_for_current_user.find(params[:snippet_id]) present @snippet, with: Entities::ProjectSnippet end @@ -73,7 +78,7 @@ module API # Example Request: # PUT /projects/:id/snippets/:snippet_id put ":id/snippets/:snippet_id" do - @snippet = user_project.snippets.find(params[:snippet_id]) + @snippet = snippets_for_current_user.find(params[:snippet_id]) authorize! :update_project_snippet, @snippet attrs = attributes_for_keys [:title, :file_name, :visibility_level] @@ -97,7 +102,7 @@ module API # DELETE /projects/:id/snippets/:snippet_id delete ":id/snippets/:snippet_id" do begin - @snippet = user_project.snippets.find(params[:snippet_id]) + @snippet = snippets_for_current_user.find(params[:snippet_id]) authorize! :update_project_snippet, @snippet @snippet.destroy rescue @@ -113,7 +118,7 @@ module API # Example Request: # GET /projects/:id/snippets/:snippet_id/raw get ":id/snippets/:snippet_id/raw" do - @snippet = user_project.snippets.find(params[:snippet_id]) + @snippet = snippets_for_current_user.find(params[:snippet_id]) env['api.format'] = :txt content_type 'text/plain' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6fcb5261e40..5a22d14988f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -44,7 +44,7 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.starred_projects + @projects = current_user.viewable_starred_projects @projects = filter_projects(@projects) @projects = paginate @projects present @projects, with: Entities::Project @@ -94,6 +94,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # namespace_id (optional) - defaults to user namespace # public (optional) - if true same as setting visibility_level = 20 @@ -112,6 +113,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :namespace_id, :public, @@ -143,6 +145,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) @@ -206,6 +209,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project @@ -222,6 +226,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :public, :visibility_level, @@ -244,6 +249,68 @@ module API end end + # Archive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/archive + post ':id/archive' do + authorize!(:archive_project, user_project) + + user_project.archive! + + present user_project, with: Entities::Project + end + + # Unarchive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/unarchive + post ':id/unarchive' do + authorize!(:archive_project, user_project) + + user_project.unarchive! + + present user_project, with: Entities::Project + end + + # Star project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/star + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + end + end + + # Unstar project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id/star + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + else + not_modified! + end + end + # Remove project # # Parameters: diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0d0f0d4616d..f55aceed92c 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -56,8 +56,7 @@ module API blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) not_found! "File" unless blob - content_type 'text/plain' - header *Gitlab::Workhorse.send_git_blob(repo, blob) + send_git_blob repo, blob end # Get a raw blob contents by blob sha @@ -80,10 +79,7 @@ module API not_found! 'Blob' unless blob - env['api.format'] = :txt - - content_type blob.mime_type - header *Gitlab::Workhorse.send_git_blob(repo, blob) + send_git_blob repo, blob end # Get a an archive of the repository @@ -98,8 +94,7 @@ module API authorize! :download_code, user_project begin - RepositoryArchiveCacheWorker.perform_async - header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) + send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue not_found!('File') end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 8ec91485b26..4faba9dc87b 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list] + attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else diff --git a/lib/api/session.rb b/lib/api/session.rb index cc646895914..56c202f1294 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -11,8 +11,7 @@ module API # Example Request: # POST /session post "/session" do - auth = Gitlab::Auth.new - user = auth.find(params[:email] || params[:login], params[:password]) + user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user present user, with: Entities::UserLogin diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb new file mode 100644 index 00000000000..c49e2a21b82 --- /dev/null +++ b/lib/api/subscriptions.rb @@ -0,0 +1,60 @@ +module API + class Subscriptions < Grape::API + before { authenticate! } + + subscribable_types = { + 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'issues' => proc { |id| find_project_issue(id) }, + 'labels' => proc { |id| find_project_label(id) }, + } + + resource :projects do + subscribable_types.each do |type, finder| + type_singularized = type.singularize + type_id_str = :"#{type_singularized}_id" + entity_class = Entities.const_get(type_singularized.camelcase) + + # Subscribe to a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # POST /projects/:id/labels/:subscribable_id/subscription + # POST /projects/:id/issues/:subscribable_id/subscription + # POST /projects/:id/merge_requests/:subscribable_id/subscription + post ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if resource.subscribed?(current_user) + not_modified! + else + resource.subscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + + # Unsubscribe from a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # DELETE /projects/:id/labels/:subscribable_id/subscription + # DELETE /projects/:id/issues/:subscribable_id/subscription + # DELETE /projects/:id/merge_requests/:subscribable_id/subscription + delete ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if !resource.subscribed?(current_user) + not_modified! + else + resource.unsubscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + end + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 2d8a9e51bb9..3e1ed3fe5c7 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -12,10 +12,24 @@ module API # Example Request: # GET /projects/:id/repository/tags get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, + present user_project.repository.tags.sort_by(&:name).reverse, with: Entities::RepoTag, project: user_project end + # Get a single repository tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # GET /projects/:id/repository/tags/:tag_name + get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + present tag, with: Entities::RepoTag, project: user_project + end + # Create tag # # Parameters: diff --git a/lib/api/users.rb b/lib/api/users.rb index 13ab17c6904..8a376d3c2a3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -11,6 +11,10 @@ module API # GET /users?search=Admin # GET /users?username=root get do + unless can?(current_user, :read_users_list, nil) + render_api_error!("Not authorized.", 403) + end + if params[:username].present? @users = User.where(username: params[:username]) else @@ -36,10 +40,12 @@ module API get ":id" do @user = User.find(params[:id]) - if current_user.is_admin? + if current_user && current_user.is_admin? present @user, with: Entities::UserFull - else + elsif can?(current_user, :read_user, @user) present @user, with: Entities::User + else + render_api_error!("User not found.", 404) end end @@ -58,6 +64,7 @@ module API # extern_uid - External authentication provider UID # provider - External provider # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # confirm - Require user confirmation - true (default) or false @@ -67,9 +74,9 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) + confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) user.admin = admin unless admin.nil? user.skip_confirmation! unless confirm @@ -106,6 +113,7 @@ module API # website_url - Website url # projects_limit - Limit projects each user can create # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # external - Flags the user as external - true or false(default) @@ -114,7 +122,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb deleted file mode 100644 index 783fcfb61ad..00000000000 --- a/lib/award_emoji.rb +++ /dev/null @@ -1,51 +0,0 @@ -class AwardEmoji - CATEGORIES = { - other: "Other", - objects: "Objects", - places: "Places", - travel_places: "Travel", - emoticons: "Emoticons", - objects_symbols: "Symbols", - nature: "Nature", - celebration: "Celebration", - people: "People", - activity: "Activity", - flags: "Flags", - food_drink: "Food" - }.with_indifferent_access - - def self.normilize_emoji_name(name) - aliases[name] || name - end - - def self.emoji_by_category - unless @emoji_by_category - @emoji_by_category = {} - - emojis.each do |emoji_name, data| - data["name"] = emoji_name - - @emoji_by_category[data["category"]] ||= [] - @emoji_by_category[data["category"]] << data - end - - @emoji_by_category = @emoji_by_category.sort.to_h - end - - @emoji_by_category - end - - def self.emojis - @emojis ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) - JSON.parse(File.read(json_path)) - end - end - - def self.aliases - @aliases ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) - JSON.parse(File.read(json_path)) - end - end -end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 67b2a64bd10..22319ec6623 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -86,9 +86,9 @@ module Backup def report_success(success) if success - $progress.puts '[DONE]'.green + $progress.puts '[DONE]'.color(:green) else - $progress.puts '[FAILED]'.red + $progress.puts '[FAILED]'.color(:red) end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..2ff3e3bdfb0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,5 +1,8 @@ module Backup class Manager + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] + FOLDERS_TO_BACKUP = %w[repositories db] + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -24,9 +27,9 @@ module Backup # Set file permissions on open to prevent chmod races. tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]} if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "creating archive #{tar_file} failed".red + puts "creating archive #{tar_file} failed".color(:red) abort 'Backup failed' end @@ -35,24 +38,22 @@ module Backup end def upload(tar_file) - remote_directory = Gitlab.config.backup.upload.remote_directory $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - $progress.puts "skipped".yellow + $progress.puts "skipped".color(:yellow) return end - connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) + directory = connect_to_remote_directory(connection_settings) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "uploading backup to #{remote_directory} failed".red + puts "uploading backup to #{remote_directory} failed".color(:red) abort 'Backup failed' end end @@ -64,9 +65,9 @@ module Backup next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "deleting tmp directory '#{dir}' failed".red + puts "deleting tmp directory '#{dir}' failed".color(:red) abort 'Backup failed' end end @@ -92,9 +93,9 @@ module Backup end end - $progress.puts "done. (#{removed} removed)".green + $progress.puts "done. (#{removed} removed)".color(:green) else - $progress.puts "skipping".yellow + $progress.puts "skipping".color(:yellow) end end @@ -121,20 +122,20 @@ module Backup $progress.print "Unpacking backup ... " unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red + puts "unpacking backup failed".color(:red) exit 1 else - $progress.puts "done".green + $progress.puts "done".color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".red - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".red - puts " Please switch to the following version and try again:".red - puts " version: #{settings[:gitlab_version]}".red + puts "GitLab version mismatch:".color(:red) + puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + puts " Please switch to the following version and try again:".color(:red) + puts " version: #{settings[:gitlab_version]}".color(:red) puts puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 @@ -147,21 +148,44 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) + settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item) end private + def connect_to_remote_directory(connection_settings) + connection = ::Fog::Storage.new(connection_settings) + + # We only attempt to create the directory for local backups. For AWS + # and other cloud providers, we cannot guarantee the user will have + # permission to create the bucket. + if connection.service == ::Fog::Storage::Local + connection.directories.create(key: remote_directory) + else + connection.directories.get(remote_directory) + end + end + + def remote_directory + Gitlab.config.backup.upload.remote_directory + end + def backup_contents folders_to_backup + archives_to_backup + ["backup_information.yml"] end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - %w{repositories db}.reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } + end + + def disabled_features + features = [] + features << 'registry' unless Gitlab.config.registry.enabled + features end def settings diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb new file mode 100644 index 00000000000..67fe0231087 --- /dev/null +++ b/lib/backup/registry.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Registry < Files + def initialize + super('registry', Settings.registry.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index a82a7e1f7bf..7b91215d50b 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -14,14 +14,14 @@ module Backup FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace if project.empty_repo? - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".green + $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".red + puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" puts output abort 'Backup failed' @@ -33,14 +33,14 @@ module Backup if File.exists?(path_to_repo(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? - $progress.puts " [SKIPPED]".cyan + $progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Backup failed' end @@ -71,9 +71,9 @@ module Backup end if system(*cmd, silent) - $progress.puts "[DONE]".green + $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".red + puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end @@ -90,21 +90,21 @@ module Backup cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) if system(*cmd, silent) - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end end end - $progress.print 'Put GitLab hooks in repositories dirs'.yellow + $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" if system(cmd) - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd}" end diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index 905c4c0144e..3eb544dfef9 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,3 @@ -require 'active_support/core_ext/string/output_safety' - module Banzai module Filter def self.[](name) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 34c38913474..4815bafe238 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -11,15 +11,15 @@ module Banzai end def self.object_name - object_class.name.underscore + @object_name ||= object_class.name.underscore end def self.object_sym - object_name.to_sym + @object_sym ||= object_name.to_sym end - def self.data_reference - "data-#{object_name.dasherize}" + def self.object_class_title + @object_title ||= object_class.name.titleize end # Public: Find references in text (like `!123` for merge requests) @@ -41,10 +41,6 @@ module Banzai end end - def self.referenced_by(node) - { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } - end - def object_class self.class.object_class end @@ -53,6 +49,10 @@ module Banzai self.class.object_sym end + def object_class_title + self.class.object_class_title + end + def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -62,36 +62,81 @@ module Banzai # Example: project.merge_requests.find end + def find_object_cached(project, id) + if RequestStore.active? + cache = find_objects_cache[object_class][project.id] + + get_or_set_cache(cache, id) { find_object(project, id) } + else + find_object(project, id) + end + end + + def project_from_ref_cache(ref) + if RequestStore.active? + cache = project_refs_cache + + get_or_set_cache(cache, ref) { project_from_ref(ref) } + else + project_from_ref(ref) + end + end + def url_for_object(object, project) # Implement in child class # Example: project_merge_request_url end - def call - if object_class.reference_pattern - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + def url_for_object_cached(object, project) + if RequestStore.active? + cache = url_for_object_cache[object_class][project.id] - # `[Issue](#123)`, which is turned into - # `<a href="#123">Issue</a>` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) - end + get_or_set_cache(cache, object) { url_for_object(object, project) } + else + url_for_object(object, project) end + end - if object_class.link_reference_pattern - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + def call + return doc if project.nil? + + ref_pattern = object_class.reference_pattern + link_pattern = object_class.link_reference_pattern + + nodes.each do |node| + if text_node?(node) && ref_pattern + replace_text_when_pattern_matches(node, ref_pattern) do |content| + object_link_filter(content, ref_pattern) + end + + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if ref_pattern && link =~ /\A#{ref_pattern}\z/ + replace_link_node_with_href(node, link) do + object_link_filter(link, ref_pattern, link_text: text) + end + + next + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + next unless link_pattern + + if link == text && text =~ /\A#{link_pattern}/ + replace_link_node_with_text(node, link) do + object_link_filter(text, link_pattern) + end + + next + end + + if link =~ /\A#{link_pattern}\z/ + replace_link_node_with_href(node, link) do + object_link_filter(link, link_pattern, link_text: text) + end + + next + end + end end end @@ -109,9 +154,9 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_text: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref(project_ref) + project = project_from_ref_cache(project_ref) - if project && object = find_object(project, id) + if project && object = find_object_cached(project, id) title = object_link_title(object) klass = reference_class(object_sym) @@ -121,8 +166,11 @@ module Banzai object_sym => object.id ) - url = matches[:url] if matches.names.include?("url") - url ||= url_for_object(object, project) + if matches.names.include?("url") && matches[:url] + url = matches[:url] + else + url = url_for_object_cached(object, project) + end text = link_text || object_link_text(object, matches) @@ -146,7 +194,7 @@ module Banzai end def object_link_title(object) - "#{object_class.name.titleize}: #{object.title}" + "#{object_class_title}: #{object.title}" end def object_link_text(object, matches) @@ -157,6 +205,83 @@ module Banzai text end + + # Returns a Hash containing all object references (e.g. issue IDs) per the + # project they belong to. + def references_per_project + @references_per_project ||= begin + refs = Hash.new { |hash, key| hash[key] = Set.new } + + regex = Regexp.union(object_class.reference_pattern, + object_class.link_reference_pattern) + + nodes.each do |node| + node.to_html.scan(regex) do + project = $~[:project] || current_project_path + + refs[project] << $~[object_sym] + end + end + + refs + end + end + + # Returns a Hash containing referenced projects grouped per their full + # path. + def projects_per_reference + @projects_per_reference ||= begin + hash = {} + refs = Set.new + + references_per_project.each do |project_ref, _| + refs << project_ref + end + + find_projects_for_paths(refs.to_a).each do |project| + hash[project.path_with_namespace] = project + end + + hash + end + end + + # Returns the projects for the given paths. + def find_projects_for_paths(paths) + Project.where_paths_in(paths).includes(:namespace) + end + + def current_project_path + @current_project_path ||= project.path_with_namespace + end + + private + + def project_refs_cache + RequestStore[:banzai_project_refs] ||= {} + end + + def find_objects_cache + RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end + + def url_for_object_cache + RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end + + def get_or_set_cache(cache, key) + if cache.key?(key) + cache[key] + else + value = yield + cache[key] = value if key.present? + value + end + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 856f56fb175..fac7dad3243 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 470727ee312..bbb88c979cc 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit_range + def self.object_class CommitRange end @@ -14,36 +16,20 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit-range") - range = find_object(project, id) - - return unless range - - { commit_range: range } - end - def initialize(*args) super @commit_map = {} end - def self.find_object(project, id) + def find_object(project, id) range = CommitRange.new(id, project) range.valid_commits? ? range : nil end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(range, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 713a56ba949..2ce1816672b 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit + def self.object_class Commit end @@ -14,30 +16,14 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit") - commit = find_object(project, id) - - return unless commit - - { commit: commit } - end - - def self.find_object(project, id) + def find_object(project, id) if project && project.valid_repo? project.commit(id) end end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(commit, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 207437ba7cf..d25de900674 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,7 +1,3 @@ -require 'action_controller' -require 'gitlab_emoji' -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that replaces :emoji: with images. diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index edc26386903..eaa702952cc 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # References are ignored if the project doesn't use an external issue # tracker. class ExternalIssueReferenceFilter < ReferenceFilter + self.reference_type = :external_issue + # Public: Find `JIRA-123` issue references in text # # ExternalIssueReferenceFilter.references_in(text) do |match, issue| @@ -21,29 +23,31 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-external-issue") - external_issue = ExternalIssue.new(id, project) - - return unless external_issue - - { external_issue: external_issue } - end - def call # Early return if the project isn't using an external tracker - return doc if project.nil? || project.default_issues_tracker? + return doc if project.nil? || default_issues_tracker? - replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| - issue_link_filter(content) - end + ref_pattern = ExternalIssue.reference_pattern + ref_start_pattern = /\A#{ref_pattern}\z/ + + each_node do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + issue_link_filter(content) + end - replace_link_nodes_with_href(ExternalIssue.reference_pattern) do |link, text| - issue_link_filter(link, link_text: text) + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_start_pattern + replace_link_node_with_href(node, link) do + issue_link_filter(link, link_text: text) + end + end + end + end end + + doc end # Replace `JIRA-123` issue references in text with links to the referenced @@ -76,6 +80,21 @@ module Banzai def url_for_issue(*args) IssuesHelper.url_for_issue(*args) end + + def default_issues_tracker? + if RequestStore.active? + default_issues_tracker_cache[project.id] ||= + project.default_issues_tracker? + else + project.default_issues_tracker? + end + end + + private + + def default_issues_tracker_cache + RequestStore[:banzai_default_issues_tracker_cache] ||= {} + end end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 8d368f3b9e7..0a29c547a4d 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,23 +1,12 @@ -require 'html/pipeline/filter' - module Banzai module Filter - # HTML Filter to add a `rel="nofollow"` attribute to external links - # + # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - doc.search('a').each do |node| - link = node.attr('href') - - next unless link - - # Skip non-HTTP(S) links - next unless link.start_with?('http') - - # Skip internal links - next if link.start_with?(internal_url) - - node.set_attribute('rel', 'nofollow') + # Skip non-HTTP(S) links and internal links + doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| + node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') end doc diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f31f921903b..d08267a9d6c 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -1,6 +1,3 @@ -require 'banzai' -require 'html/pipeline/filter' - module Banzai module Filter # HTML Filter for parsing Gollum's tags in HTML. It's only parses the @@ -121,7 +118,7 @@ module Banzai end if path - content_tag(:img, nil, src: path) + content_tag(:img, nil, src: path, class: 'gfm') end end @@ -147,12 +144,18 @@ module Banzai # if it is not. def process_page_link_tag(parts) if parts.size == 1 - url = parts[0].strip + reference = parts[0].strip + else + name, reference = *parts.compact.map(&:strip) + end + + if url?(reference) + href = reference else - name, url = *parts.compact.map(&:strip) + href = ::File.join(project_wiki_base_path, reference) end - content_tag(:a, name || url, href: url) + content_tag(:a, name || reference, href: href, class: 'gfm') end def project_wiki diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb new file mode 100644 index 00000000000..ccd106860bd --- /dev/null +++ b/lib/banzai/filter/image_link_filter.rb @@ -0,0 +1,27 @@ +module Banzai + module Filter + # HTML filter that wraps links around inline images. + class ImageLinkFilter < HTML::Pipeline::Filter + + # Find every image that isn't already wrapped in an `a` tag, create + # a new node (a link to the image source), copy the image as a child + # of the anchor, and then replace the img with the link-wrapped version. + def call + doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + + link = doc.document.create_element( + 'a', + class: 'no-attachment-icon', + href: img['src'], + target: '_blank' + ) + + link.children = img.clone + img.replace(link) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb new file mode 100644 index 00000000000..beb21b19ab3 --- /dev/null +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -0,0 +1,26 @@ +module Banzai + module Filter + class InlineDiffFilter < HTML::Pipeline::Filter + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + + def call + search_text_nodes(doc).each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + content = node.to_html + html_content = inline_diff_filter(content) + + next if content == html_content + + node.replace(html_content) + end + doc + end + + def inline_diff_filter(text) + html_content = text.gsub(/(?:\[\-(.*?)\-\]|\{\-(.*?)\-\})/, '<span class="idiff left right deletion">\1\2</span>') + html_content.gsub(/(?:\[\+(.*?)\+\]|\{\+(.*?)\+\})/, '<span class="idiff left right addition">\1\2</span>') + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 9f08aa36e8b..2614261f9eb 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -5,17 +5,46 @@ module Banzai # # This filter supports cross-project references. class IssueReferenceFilter < AbstractReferenceFilter + self.reference_type = :issue + def self.object_class Issue end - def find_object(project, id) - project.get_issue(id) + def find_object(project, iid) + issues_per_project[project][iid] end def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) end + + def project_from_ref(ref) + projects_per_reference[ref || current_project_path] + end + + # Returns a Hash containing the issues per Project instance. + def issues_per_project + @issues_per_project ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + projects_per_reference.each do |path, project| + issue_ids = references_per_project[path] + + next unless project.default_issues_tracker? + + project.issues.where(iid: issue_ids.to_a).each do |issue| + hash[project][issue.iid] = issue + end + end + + hash + end + end + + def find_projects_for_paths(paths) + super(paths).includes(:gitlab_issue_tracker_service) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8147e5ed3c7..e4d3f87d0aa 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -2,6 +2,8 @@ module Banzai module Filter # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter + self.reference_type = :label + def self.object_class Label end @@ -18,9 +20,7 @@ module Banzai def references_in(text, pattern = Label.reference_pattern) text.gsub(pattern) do |match| - project = project_from_ref($~[:project]) - params = label_params($~[:label_id].to_i, $~[:label_name]) - label = project.labels.find_by(params) + label = find_label($~[:project], $~[:label_id], $~[:label_name]) if label yield match, label.id, $~[:project], $~ @@ -30,18 +30,12 @@ module Banzai end end - def url_for_object(label, project) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_issues_url(project.namespace, project, label_name: label.name, - only_path: context[:only_path]) - end + def find_label(project_ref, label_id, label_name) + project = project_from_ref(project_ref) + return unless project - def object_link_text(object, matches) - if context[:project] == object.project - LabelsHelper.render_colored_label(object) - else - LabelsHelper.render_colored_cross_project_label(object) - end + label_params = label_params(label_id, label_name) + project.labels.find_by(label_params) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -55,7 +49,21 @@ module Banzai if name { name: name.tr('"', '') } else - { id: id } + { id: id.to_i } + end + end + + def url_for_object(label, project) + h = Gitlab::Routing.url_helpers + h.namespace_project_issues_url(project.namespace, project, label_name: label.name, + only_path: context[:only_path]) + end + + def object_link_text(object, matches) + if context[:project] == object.project + LabelsHelper.render_colored_label(object) + else + LabelsHelper.render_colored_cross_project_label(object) end end end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 0659fed1419..9b209533a89 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 57c71708992..ac5216d9cfb 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class MergeRequestReferenceFilter < AbstractReferenceFilter + self.reference_type = :merge_request + def self.object_class MergeRequest end @@ -14,7 +16,7 @@ module Banzai end def url_for_object(mr, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index e88b27c1fae..ca686c87d97 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -1,9 +1,9 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces milestone references with links. class MilestoneReferenceFilter < AbstractReferenceFilter + self.reference_type = :milestone + def self.object_class Milestone end @@ -12,11 +12,53 @@ module Banzai project.milestones.find_by(iid: id) end - def url_for_object(issue, project) - h = Gitlab::Application.routes.url_helpers + def references_in(text, pattern = Milestone.reference_pattern) + # We'll handle here the references that follow the `reference_pattern`. + # Other patterns (for example, the link pattern) are handled by the + # default implementation. + return super(text, pattern) if pattern != Milestone.reference_pattern + + text.gsub(pattern) do |match| + milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + + if milestone + yield match, milestone.iid, $~[:project], $~ + else + match + end + end + end + + def find_milestone(project_ref, milestone_id, milestone_name) + project = project_from_ref(project_ref) + return unless project + + milestone_params = milestone_params(milestone_id, milestone_name) + project.milestones.find_by(milestone_params) + end + + def milestone_params(iid, name) + if name + { name: name.tr('"', '') } + else + { iid: iid.to_i } + end + end + + def url_for_object(milestone, project) + h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end + + def object_link_text(object, matches) + if context[:project] == object.project + super + else + "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>". + html_safe + end + end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 7141ed7c9bd..c753a84a20d 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that removes references to records that the current user does @@ -9,8 +7,11 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Querying.css(doc, 'a.gfm').each do |node| - unless user_can_see_reference?(node) + nodes = Querying.css(doc, 'a.gfm[data-reference-type]') + visible = nodes_visible_to_user(nodes) + + nodes.each do |node| + unless visible.include?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text @@ -23,20 +24,30 @@ module Banzai private - def user_can_see_reference?(node) - if node.has_attribute?('data-reference-filter') - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, current_user) - reference_filter.user_can_see_reference?(current_user, node, context) - else - true + visible.merge(parser.nodes_visible_to_user(current_user, nodes)) end + + visible end def current_user context[:current_user] end + + def project + context[:project] + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 3637b1bac94..2d6f34c9cd8 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,6 +1,3 @@ -require 'active_support/core_ext/string/output_safety' -require 'html/pipeline/filter' - module Banzai module Filter # Base class for GitLab Flavored Markdown reference filters. @@ -11,24 +8,8 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-project') - project_id = node.attr('data-project').to_i - return true if project_id == context[:project].try(:id) - - project = Project.find(project_id) rescue nil - Ability.abilities.allowed?(user, :read_project, project) - else - true - end - end - - def self.user_can_reference?(user, node, context) - true - end - - def self.referenced_by(node) - raise NotImplementedError, "#{self} does not implement #{__method__}" + class << self + attr_accessor :reference_type end # Returns a data attribute String to attach to a reference link @@ -46,7 +27,10 @@ module Banzai # # Returns a String def data_attribute(attributes = {}) - attributes[:reference_filter] = self.class.name.demodulize + attributes = attributes.reject { |_, v| v.nil? } + + attributes[:reference_type] = self.class.reference_type + attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end @@ -54,18 +38,13 @@ module Banzai html.html_safe? ? html : ERB::Util.html_escape_once(html) end - def ignore_parents - @ignore_parents ||= begin - # Don't look for references in text nodes that are children of these - # elements. + def ignore_ancestor_query + @ignore_ancestor_query ||= begin parents = %w(pre code a style) parents << 'blockquote' if context[:ignore_blockquotes] - parents.to_set - end - end - def ignored_ancestry?(node) - has_ancestor?(node, ignore_parents) + parents.map { |n| "ancestor::#{n}" }.join(' or ') + end end def project @@ -76,119 +55,73 @@ module Banzai "gfm gfm-#{type}" end - # Iterate through the document's text nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` AND - # * The node is not an ancestor of an ignored node type - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node's existing content and update the current document. + # Ensure that a :project key exists in context # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_text_nodes_matching(pattern) - return doc if project.nil? - - search_text_nodes(doc).each do |node| - next if ignored_ancestry?(node) - next unless node.text =~ pattern - - content = node.to_html - - html = yield content - - next if html == content - - node.replace(html) - end - - doc + # Note that while the key might exist, its value could be nil! + def validate + needs :project end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` - # - # pattern - Regex pattern against which to match the node's content + # Iterates over all <a> and text() nodes in a document. # - # Yields the current node's String contents. The result of the block will - # replace the node and update the current document. - # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_text(pattern) - return doc if project.nil? - - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') - - link = node.attr('href') - text = node.text + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + return to_enum(__method__) unless block_given? - next unless link && text + query = %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] + | descendant-or-self::a[ + not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") + ]} - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - # Ignore ending punctionation like periods or commas - next unless link == text && text =~ /\A#{pattern}/ + doc.xpath(query).each do |node| + yield node + end + end - html = yield text + # Returns an Array containing all HTML nodes. + def nodes + @nodes ||= each_node.to_a + end - next if html == text + # Yields the link's URL and text whenever the node is a valid <a> tag. + def yield_valid_link(node) + link = CGI.unescape(node.attr('href').to_s) + text = node.text - node.replace(html) - end + return unless link.force_encoding('UTF-8').valid_encoding? - doc + yield link, text end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's HREF matches `pattern` - # - # pattern - Regex pattern against which to match the node's HREF - # - # Yields the current node's String HREF and String content. - # The result of the block will replace the node and update the current document. - # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_href(pattern) - return doc if project.nil? + def replace_text_when_pattern_matches(node, pattern) + return unless node.text =~ pattern - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') + content = node.to_html + html = yield content - link = node.attr('href') - text = node.text + node.replace(html) unless content == html + end - next unless link && text - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - next unless link && link =~ /\A#{pattern}\z/ + def replace_link_node_with_text(node, link) + html = yield - html = yield link, text + node.replace(html) unless html == node.text + end - next if html == link + def replace_link_node_with_href(node, link) + html = yield - node.replace(html) - end + node.replace(html) unless html == link + end - doc + def text_node?(node) + node.is_a?(Nokogiri::XML::Text) end - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project + def element_node?(node) + node.is_a?(Nokogiri::XML::Element) end end end diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb deleted file mode 100644 index 86d484feb90..00000000000 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'html/pipeline/filter' - -module Banzai - module Filter - # HTML filter that gathers all referenced records that the current user has - # permission to view. - # - # Expected to be run in its own post-processing pipeline. - # - class ReferenceGathererFilter < HTML::Pipeline::Filter - def initialize(*) - super - - result[:references] ||= Hash.new { |hash, type| hash[type] = [] } - end - - def call - Querying.css(doc, 'a.gfm').each do |node| - gather_references(node) - end - - load_lazy_references unless ReferenceExtractor.lazy? - - doc - end - - private - - def gather_references(node) - return unless node.has_attribute?('data-reference-filter') - - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) - - return if context[:reference_filter] && reference_filter != context[:reference_filter] - - return if author && !reference_filter.user_can_reference?(author, node, context) - - return unless reference_filter.user_can_see_reference?(current_user, node, context) - - references = reference_filter.referenced_by(node) - return unless references - - references.each do |type, values| - Array.wrap(values).each do |value| - result[:references][type] << value - end - end - end - - def load_lazy_references - refs = result[:references] - refs.each do |type, values| - refs[type] = ReferenceExtractor.lazily(values) - end - end - - def current_user - context[:current_user] - end - - def author - context[:author] - end - end - end -end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 41380627d39..ea21c7b041c 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index e8011519608..ca80aac5a08 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'html/pipeline/sanitization_filter' - module Banzai module Filter # Sanitize HTML @@ -66,7 +63,7 @@ module Banzai begin uri = Addressable::URI.parse(node['href']) - uri.scheme.strip! if uri.scheme + uri.scheme = uri.scheme.strip.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index c870a42f741..212a0bbf2a0 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter + self.reference_type = :snippet + def self.object_class Snippet end @@ -14,7 +16,7 @@ module Banzai end def url_for_object(snippet, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 8c5855e5ffc..62a79c62e20 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' module Banzai diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 4056dcd6d64..a4eda6fdf76 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that adds an anchor child element to all Headers in a diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index f642aee0967..45bb66dc99f 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai @@ -9,11 +8,13 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - doc.search('a').each do |el| + return doc unless project + + doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') end - doc.search('img').each do |el| + doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el| process_link_attr el.attribute('src') end @@ -23,16 +24,15 @@ module Banzai protected def process_link_attr(html_attr) - return if html_attr.blank? - - uri = html_attr.value - if uri.starts_with?("/uploads/") - html_attr.value = build_url(uri).to_s - end + html_attr.value = build_url(html_attr.value).to_s end def build_url(uri) - File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) + end + + def project + context[:project] end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 24f16f8b547..5b0a6d8541b 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter + self.reference_type = :user + # Public: Find `@user` user references in text # # UserReferenceFilter.references_in(text) do |match, username| @@ -21,51 +23,29 @@ module Banzai end end - def self.referenced_by(node) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - return unless group - - { user: group.users } - elsif node.has_attribute?('data-user') - { user: LazyReference.new(User, node.attr('data-user')) } - elsif node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return unless project - - { user: project.team.members.flatten } - end - end - - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - Ability.abilities.allowed?(user, :read_group, group) - else - super - end - end - - def self.user_can_reference?(user, node, context) - # Only team members can reference `@all` - if node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return false unless project - - user && project.team.member?(user) - else - super - end - end - def call - replace_text_nodes_matching(User.reference_pattern) do |content| - user_link_filter(content) + return doc if project.nil? + + ref_pattern = User.reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + nodes.each do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + user_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_pattern_start + replace_link_node_with_href(node, link) do + user_link_filter(link, link_text: text) + end + end + end + end end - replace_link_nodes_with_href(User.reference_pattern) do |link, text| - user_link_filter(link, link_text: text) - end + doc end # Replace `@user` user references in text with links to the referenced @@ -79,7 +59,7 @@ module Banzai self.class.references_in(text) do |match, username| if username == 'all' link_to_all(link_text: link_text) - elsif namespace = Namespace.find_by(path: username) + elsif namespace = namespaces[username] link_to_namespace(namespace, link_text: link_text) || match else match @@ -87,10 +67,35 @@ module Banzai end end + # Returns a Hash containing all Namespace objects for the username + # references in the current document. + # + # The keys of this Hash are the namespace paths, the values the + # corresponding Namespace objects. + def namespaces + @namespaces ||= + Namespace.where(path: usernames).each_with_object({}) do |row, hash| + hash[row.path] = row + end + end + + # Returns all usernames referenced in the current document. + def usernames + refs = Set.new + + nodes.each do |node| + node.to_html.scan(User.reference_pattern) do + refs << $~[:user] + end + end + + refs.to_a + end + private def urls - Gitlab::Application.routes.url_helpers + Gitlab::Routing.url_helpers end def link_class @@ -99,9 +104,12 @@ module Banzai def link_to_all(link_text: nil) project = context[:project] + author = context[:author] + url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) - data = data_attribute(project: project.id) + + data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' link_tag(url, data, text) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb new file mode 100644 index 00000000000..37a2779d453 --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -0,0 +1,41 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" links to pages/files in a wiki. + # Rewrite rules are documented in the `WikiPipeline` spec. + # + # Context options: + # :project_wiki + class WikiLinkFilter < HTML::Pipeline::Filter + + def call + return doc unless project_wiki? + + doc.search('a:not(.gfm)').each do |el| + process_link_attr el.attribute('href') + end + + doc + end + + protected + + def project_wiki? + !context[:project_wiki].nil? + end + + def process_link_attr(html_attr) + return if html_attr.blank? + + html_attr.value = apply_rewrite_rules(html_attr.value) + rescue URI::Error + # noop + end + + def apply_rewrite_rules(link_string) + Rewriter.new(link_string, wiki: context[:project_wiki], slug: context[:page_slug]).apply_rules + end + end + end +end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb new file mode 100644 index 00000000000..2e2c8da311e --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -0,0 +1,40 @@ +module Banzai + module Filter + class WikiLinkFilter < HTML::Pipeline::Filter + class Rewriter + def initialize(link_string, wiki:, slug:) + @uri = Addressable::URI.parse(link_string) + @wiki_base_path = wiki && wiki.wiki_base_path + @slug = slug + end + + def apply_rules + apply_file_link_rules! + apply_hierarchical_link_rules! + apply_relative_link_rules! + @uri.to_s + end + + private + + # Of the form 'file.md' + def apply_file_link_rules! + @uri = Addressable::URI.join(@slug, @uri) if @uri.extname.present? + end + + # Of the form `./link`, `../link`, or similar + def apply_hierarchical_link_rules! + @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' + end + + # Any link _not_ of the form `http://example.com/` + def apply_relative_link_rules! + if @uri.relative? && @uri.path.present? + link = ::File.join(@wiki_base_path, @uri.path) + @uri = Addressable::URI.parse(link) + end + end + end + end + end +end diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb index e4e2f3f228d..58e3e81209e 100644 --- a/lib/banzai/filter/yaml_front_matter_filter.rb +++ b/lib/banzai/filter/yaml_front_matter_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'yaml' - module Banzai module Filter class YamlFrontMatterFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb deleted file mode 100644 index 1095b4debc7..00000000000 --- a/lib/banzai/lazy_reference.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Banzai - class LazyReference - def self.load(refs) - lazy_references, values = refs.partition { |ref| ref.is_a?(self) } - - lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| - ids = refs.flat_map(&:ids) - klass.where(id: ids) - end - - values + lazy_values - end - - attr_reader :klass, :ids - - def initialize(klass, ids) - @klass = klass - @ids = Array.wrap(ids).map(&:to_i) - end - - def load - self.klass.where(id: self.ids) - end - end -end diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index f60966c3c0f..321fd5bbe14 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,5 +1,3 @@ -require 'html/pipeline' - module Banzai module Pipeline class BasePipeline diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index f2395867658..042fb2e6e14 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,23 +1,16 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline + WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) + ) + def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: whitelist + whitelist: WHITELIST ) end - - private - - def self.whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - whitelist = Banzai::Filter::SanitizationFilter::LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - - whitelist - end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8cd4b50e65a..b27ecf3c923 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::UploadLinkFilter, + Filter::ImageLinkFilter, Filter::EmojiFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, @@ -22,7 +23,8 @@ module Banzai Filter::LabelReferenceFilter, Filter::MilestoneReferenceFilter, - Filter::TaskListFilter + Filter::TaskListFilter, + Filter::InlineDiffFilter ] end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb deleted file mode 100644 index 919998380e4..00000000000 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module Pipeline - class ReferenceExtractionPipeline < BasePipeline - def self.filters - FilterArray[ - Filter::ReferenceGathererFilter - ] - end - end - end -end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 9b4ff0f0f80..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -1,11 +1,11 @@ -require 'banzai' - module Banzai module Pipeline class WikiPipeline < FullPipeline def self.filters - @filters ||= super.insert_after(Filter::TableOfContentsFilter, - Filter::GollumTagsFilter) + @filters ||= begin + super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) + .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) + end end end end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index f4079538ec5..bf366962aef 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,28 +1,6 @@ module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - class << self - LAZY_KEY = :banzai_reference_extractor_lazy - - def lazy? - Thread.current[LAZY_KEY] - end - - def lazily(values = nil, &block) - return (values || block.call).uniq if lazy? - - begin - Thread.current[LAZY_KEY] = true - - values ||= block.call - - Banzai::LazyReference.load(values.uniq).uniq - ensure - Thread.current[LAZY_KEY] = false - end - end - end - def initialize @texts = [] end @@ -31,23 +9,21 @@ module Banzai @texts << Renderer.render(text, context) end - def references(type, context = {}) - filter = Banzai::Filter["#{type}_reference"] + def references(type, project, current_user = nil) + processor = Banzai::ReferenceParser[type]. + new(project, current_user) + + processor.process(html_documents) + end - context.merge!( - pipeline: :reference_extraction, + private - # ReferenceGathererFilter - reference_filter: filter - ) + def html_documents + # This ensures that we don't memoize anything until we have a number of + # text blobs to parse. + return [] if @texts.empty? - self.class.lazily do - @texts.flat_map do |html| - text_context = context.dup - result = Renderer.render_result(html, text_context) - result[:references][type] - end.uniq - end + @html_documents ||= @texts.map { |html| Nokogiri::HTML.fragment(html) } end end end diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..557bec4316e --- /dev/null +++ b/lib/banzai/reference_parser.rb @@ -0,0 +1,14 @@ +module Banzai + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['issue'] + # + # This would return the `Banzai::ReferenceParser::IssueParser` class. + def self.[](name) + const_get("#{name.to_s.camelize}Parser") + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb new file mode 100644 index 00000000000..3d7b9c4a024 --- /dev/null +++ b/lib/banzai/reference_parser/base_parser.rb @@ -0,0 +1,204 @@ +module Banzai + module ReferenceParser + # Base class for reference parsing classes. + # + # Each parser should also specify its reference type by calling + # `self.reference_type = ...` in the body of the class. The value of this + # method should be a symbol such as `:issue` or `:merge_request`. For + # example: + # + # class IssueParser < BaseParser + # self.reference_type = :issue + # end + # + # The reference type is used to determine what nodes to pass to the + # `referenced_by` method. + # + # Parser classes should either implement the instance method + # `references_relation` or overwrite `referenced_by`. The + # `references_relation` method is supposed to return an + # ActiveRecord::Relation used as a base relation for retrieving the objects + # referenced in a set of HTML nodes. + # + # Each class can implement two additional methods: + # + # * `nodes_user_can_reference`: returns an Array of nodes the given user can + # refer to. + # * `nodes_visible_to_user`: returns an Array of nodes that are visible to + # the given user. + # + # You only need to overwrite these methods if you want to tweak who can see + # which references. For example, the IssueParser class defines its own + # `nodes_visible_to_user` method so it can ensure users can only see issues + # they have access to. + class BaseParser + class << self + attr_accessor :reference_type + end + + # Returns the attribute name containing the value for every object to be + # parsed by the current parser. + # + # For example, for a parser class that returns "Animal" objects this + # attribute would be "data-animal". + def self.data_attribute + @data_attribute ||= "data-#{reference_type.to_s.dasherize}" + end + + def initialize(project = nil, current_user = nil) + @project = project + @current_user = current_user + end + + # Returns all the nodes containing references that the user can refer to. + def nodes_user_can_reference(user, nodes) + nodes + end + + # Returns all the nodes that are visible to the given user. + def nodes_visible_to_user(user, nodes) + projects = lazy { projects_for_nodes(nodes) } + project_attr = 'data-project' + + nodes.select do |node| + if node.has_attribute?(project_attr) + node_id = node.attr(project_attr).to_i + + if project && project.id == node_id + true + else + can?(user, :read_project, projects[node_id]) + end + else + true + end + end + end + + # Returns an Array of objects referenced by any of the given HTML nodes. + def referenced_by(nodes) + ids = unique_attribute_values(nodes, self.class.data_attribute) + + references_relation.where(id: ids) + end + + # Returns the ActiveRecord::Relation to use for querying references in the + # DB. + def references_relation + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + # Returns a Hash containing attribute values per project ID. + # + # The returned Hash uses the following format: + # + # { project id => [value1, value2, ...] } + # + # nodes - An Array of HTML nodes to process. + # attribute - The name of the attribute (as a String) for which to gather + # values. + # + # Returns a Hash. + def gather_attributes_per_project(nodes, attribute) + per_project = Hash.new { |hash, key| hash[key] = Set.new } + + nodes.each do |node| + project_id = node.attr('data-project').to_i + id = node.attr(attribute) + + per_project[project_id] << id if id + end + + per_project + end + + # Returns a Hash containing objects for an attribute grouped per their + # IDs. + # + # The returned Hash uses the following format: + # + # { id value => row } + # + # nodes - An Array of HTML nodes to process. + # + # collection - The model or ActiveRecord relation to use for retrieving + # rows from the database. + # + # attribute - The name of the attribute containing the primary key values + # for every row. + # + # Returns a Hash. + def grouped_objects_for_nodes(nodes, collection, attribute) + return {} if nodes.empty? + + ids = unique_attribute_values(nodes, attribute) + + collection.where(id: ids).each_with_object({}) do |row, hash| + hash[row.id] = row + end + end + + # Returns an Array containing all unique values of an attribute of the + # given nodes. + def unique_attribute_values(nodes, attribute) + values = Set.new + + nodes.each do |node| + if node.has_attribute?(attribute) + values << node.attr(attribute) + end + end + + values.to_a + end + + # Processes the list of HTML documents and returns an Array containing all + # the references. + def process(documents) + type = self.class.reference_type + + nodes = documents.flat_map do |document| + Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a + end + + gather_references(nodes) + end + + # Gathers the references for the given HTML nodes. + def gather_references(nodes) + nodes = nodes_user_can_reference(current_user, nodes) + nodes = nodes_visible_to_user(current_user, nodes) + + referenced_by(nodes) + end + + # Returns a Hash containing the projects for a given list of HTML nodes. + # + # The returned Hash uses the following format: + # + # { project ID => project } + # + def projects_for_nodes(nodes) + @projects_for_nodes ||= + grouped_objects_for_nodes(nodes, Project, 'data-project') + end + + def can?(user, permission, subject) + Ability.abilities.allowed?(user, permission, subject) + end + + def find_projects_for_hash_keys(hash) + Project.where(id: hash.keys) + end + + private + + attr_reader :current_user, :project + + def lazy(&block) + Gitlab::Lazy.new(&block) + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb new file mode 100644 index 00000000000..0fee9d267de --- /dev/null +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -0,0 +1,34 @@ +module Banzai + module ReferenceParser + class CommitParser < BaseParser + self.reference_type = :commit + + def referenced_by(nodes) + commit_ids = commit_ids_per_project(nodes) + projects = find_projects_for_hash_keys(commit_ids) + + projects.flat_map do |project| + find_commits(project, commit_ids[project.id]) + end + end + + def commit_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_commits(project, ids) + commits = [] + + return commits unless project.valid_repo? + + ids.each do |id| + commit = project.commit(id) + + commits << commit if commit + end + + commits + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb new file mode 100644 index 00000000000..69d01f8db15 --- /dev/null +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -0,0 +1,38 @@ +module Banzai + module ReferenceParser + class CommitRangeParser < BaseParser + self.reference_type = :commit_range + + def referenced_by(nodes) + range_ids = commit_range_ids_per_project(nodes) + projects = find_projects_for_hash_keys(range_ids) + + projects.flat_map do |project| + find_ranges(project, range_ids[project.id]) + end + end + + def commit_range_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_ranges(project, range_ids) + ranges = [] + + range_ids.each do |id| + range = find_object(project, id) + + ranges << range if range + end + + ranges + end + + def find_object(project, id) + range = CommitRange.new(id, project) + + range.valid_commits? ? range : nil + end + end + end +end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb new file mode 100644 index 00000000000..a1264db2111 --- /dev/null +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class ExternalIssueParser < BaseParser + self.reference_type = :external_issue + + def referenced_by(nodes) + issue_ids = issue_ids_per_project(nodes) + projects = find_projects_for_hash_keys(issue_ids) + issues = [] + + projects.each do |project| + issue_ids[project.id].each do |id| + issues << ExternalIssue.new(id, project) + end + end + + issues + end + + def issue_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb new file mode 100644 index 00000000000..f306079d833 --- /dev/null +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -0,0 +1,54 @@ +module Banzai + module ReferenceParser + class IssueParser < BaseParser + self.reference_type = :issue + + def nodes_visible_to_user(user, nodes) + # It is not possible to check access rights for external issue trackers + return nodes if project && project.external_issue_tracker + + issues = issues_for_nodes(nodes) + + nodes.select do |node| + issue = issue_for_node(issues, node) + + issue ? can?(user, :read_issue, issue) : false + end + end + + def referenced_by(nodes) + issues = issues_for_nodes(nodes) + + nodes.map { |node| issue_for_node(issues, node) }.uniq + end + + def issues_for_nodes(nodes) + @issues_for_nodes ||= grouped_objects_for_nodes( + nodes, + Issue.all.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members + ] + } + ), + self.class.data_attribute + ) + end + + private + + def issue_for_node(issues, node) + issues[node.attr(self.class.data_attribute).to_i] + end + end + end +end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb new file mode 100644 index 00000000000..e5d1eb11d7f --- /dev/null +++ b/lib/banzai/reference_parser/label_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class LabelParser < BaseParser + self.reference_type = :label + + def references_relation + Label + end + end + end +end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb new file mode 100644 index 00000000000..c9a9ca79c09 --- /dev/null +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MergeRequestParser < BaseParser + self.reference_type = :merge_request + + def references_relation + MergeRequest.includes(:author, :assignee, :target_project) + end + end + end +end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb new file mode 100644 index 00000000000..a000ac61e5c --- /dev/null +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MilestoneParser < BaseParser + self.reference_type = :milestone + + def references_relation + Milestone + end + end + end +end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb new file mode 100644 index 00000000000..fa71b3c952a --- /dev/null +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class SnippetParser < BaseParser + self.reference_type = :snippet + + def references_relation + Snippet + end + end + end +end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb new file mode 100644 index 00000000000..a12b0d19560 --- /dev/null +++ b/lib/banzai/reference_parser/user_parser.rb @@ -0,0 +1,92 @@ +module Banzai + module ReferenceParser + class UserParser < BaseParser + self.reference_type = :user + + def referenced_by(nodes) + group_ids = [] + user_ids = [] + project_ids = [] + + nodes.each do |node| + if node.has_attribute?('data-group') + group_ids << node.attr('data-group').to_i + elsif node.has_attribute?(self.class.data_attribute) + user_ids << node.attr(self.class.data_attribute).to_i + elsif node.has_attribute?('data-project') + project_ids << node.attr('data-project').to_i + end + end + + find_users_for_groups(group_ids) | find_users(user_ids) | + find_users_for_projects(project_ids) + end + + def nodes_visible_to_user(user, nodes) + group_attr = 'data-group' + groups = lazy { grouped_objects_for_nodes(nodes, Group, group_attr) } + visible = [] + remaining = [] + + nodes.each do |node| + if node.has_attribute?(group_attr) + node_group = groups[node.attr(group_attr).to_i] + + if node_group && + can?(user, :read_group, node_group) + visible << node + end + # Remaining nodes will be processed by the parent class' + # implementation of this method. + else + remaining << node + end + end + + visible + super(current_user, remaining) + end + + def nodes_user_can_reference(current_user, nodes) + project_attr = 'data-project' + author_attr = 'data-author' + + projects = lazy { projects_for_nodes(nodes) } + users = lazy { grouped_objects_for_nodes(nodes, User, author_attr) } + + nodes.select do |node| + project_id = node.attr(project_attr) + user_id = node.attr(author_attr) + + if project && project_id && project.id == project_id.to_i + true + elsif project_id && user_id + project = projects[project_id.to_i] + user = users[user_id.to_i] + + project && user ? project.team.member?(user) : false + else + true + end + end + end + + def find_users(ids) + return [] if ids.empty? + + User.where(id: ids).to_a + end + + def find_users_for_groups(ids) + return [] if ids.empty? + + User.joins(:group_members).where(members: { source_id: ids }).to_a + end + + def find_users_for_projects(ids) + return [] if ids.empty? + + Project.where(id: ids).flat_map { |p| p.team.members.to_a } + end + end + end +end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ae714c87dc5..c14a9c4c722 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -19,8 +19,10 @@ module Banzai cache_key = full_cache_key(cache_key, context[:pipeline]) if cache_key - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + Gitlab::Metrics.measure(:banzai_cached_render) do + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) + end end else cacheless_render(text, context) @@ -64,13 +66,15 @@ module Banzai private def self.cacheless_render(text, context = {}) - result = render_result(text, context) + Gitlab::Metrics.measure(:banzai_cacheless_render) do + result = render_result(text, context) - output = result[:output] - if output.respond_to?(:to_html) - output.to_html - else - output.to_s + output = result[:output] + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index ac6d667cf8d..229050151d3 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -23,8 +23,8 @@ module Ci cross: 0x10, } - def self.convert(ansi) - Converter.new().convert(ansi) + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) end class Converter @@ -84,22 +84,38 @@ module Ci def on_107(s) set_bg_color(7, 'l') end def on_109(s) set_bg_color(9, 'l') end - def convert(ansi) - @out = "" - @n_open_tags = 0 - reset() + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] + + def convert(raw, new_state) + reset_state + restore_state(raw, new_state) if new_state.present? + + start = @offset + ansi = raw[@offset..-1] + + open_new_tag - s = StringScanner.new(ansi.gsub("<", "<")) - while(!s.eos?) + s = StringScanner.new(ansi) + until s.eos? if s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\n/) + @out << '<br>' else @out << s.scan(/./m) end + @offset += s.matched_size end close_open_tags() - @out + + { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } end def handle_sequence(s) @@ -121,6 +137,20 @@ module Ci evaluate_command_stack(commands) + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag css_classes = [] unless @fg_color.nil? @@ -138,20 +168,8 @@ module Ci css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - open_new_tag(css_classes) if css_classes.length > 0 - end + return if css_classes.empty? - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) - end - - evaluate_command_stack(stack) - end - - def open_new_tag(css_classes) @out << %{<span class="#{css_classes.join(' ')}">} @n_open_tags += 1 end @@ -163,6 +181,31 @@ module Ci end end + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(raw, new_state) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > raw.length + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) + end + end + def reset @fg_color = nil @bg_color = nil diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 4e85d2c3c74..17bb99a2ae5 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -1,9 +1,7 @@ -Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} - module Ci module API class API < Grape::API - include APIGuard + include ::API::APIGuard version 'v1', using: :path rescue_from ActiveRecord::RecordNotFound do @@ -23,15 +21,17 @@ module Ci rack_response({ 'message' => '500 Internal Server Error' }, 500) end + content_type :txt, 'text/plain' + content_type :json, 'application/json' format :json helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings - mount Builds - mount Runners - mount Triggers + mount ::Ci::API::Builds + mount ::Ci::API::Runners + mount ::Ci::API::Triggers end end end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2e9a5d311f9..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -50,6 +50,39 @@ module Ci end end + # Send incremental log update - Runners only + # + # Parameters: + # id (required) - The ID of a build + # Body: + # content of logs to append + # Headers: + # Content-Range (required) - range of content that was sent + # BUILD-TOKEN (required) - The build authorization token + # Example Request: + # PATCH /builds/:id/trace.txt + patch ":id/trace.txt" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('Build has been erased!') if build.erased? + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = build.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + build.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Build-Status', build.status + header 'Range', "0-#{build.trace_length}" + end + # Authorize artifacts uploading for build - Runners only # # Parameters: @@ -81,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -112,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b25e0e573a8..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options @@ -56,7 +57,7 @@ module Ci class TriggerRequest < Grape::Entity expose :id, :variables - expose :commit, using: Commit + expose :pipeline, using: Commit, as: :commit end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 192b1d18a51..0c41f22c7c5 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,20 +28,20 @@ module Ci post "register" do required_attributes! [:token] + attributes = { description: params[:description], + tag_list: params[:tag_list] } + + unless params[:run_untagged].nil? + attributes[:run_untagged] = params[:run_untagged] + end + runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create( - description: params[:description], - tag_list: params[:tag_list], - is_shared: true - ) + Ci::Runner.create(attributes.merge(is_shared: true)) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. - project.runners.create( - description: params[:description], - tag_list: params[:tag_list] - ) + project.runners.create(attributes) end return forbidden! unless runner diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index d53bdcbd0f2..5270108ef0f 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,11 +60,12 @@ module Ci class BuildTime < Chart def collect - commits = project.ci_commits.last(30) + commits = project.pipelines.last(30) commits.each do |commit| @labels << commit.short_sha - @build_times << (commit.duration / 60) + duration = commit.duration || 0 + @build_times << (duration / 60) end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index c89e1b51019..ed86de819eb 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -1,33 +1,39 @@ module Ci class GitlabCiYamlProcessor - class ValidationError < StandardError;end + class ValidationError < StandardError; end + + include Gitlab::Ci::Config::Node::ValidationHelpers DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies] + :dependencies, :before_script, :after_script, :variables, + :environment] + ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :before_script, :image, :services, :variables, :path, :cache + attr_reader :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = YAML.safe_load(config, [Symbol], [], true) - @path = path + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash - unless @config.is_a? Hash - raise ValidationError, "YAML should be a hash" - end - - @config = @config.deep_symbolize_keys + @path = path initial_parsing validate! + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false) - builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag)} + def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + builds.select do |build| + build[:stage] == stage && + process?(build[:only], build[:except], ref, tag, trigger_request) + end end def builds @@ -40,66 +46,83 @@ module Ci @stages || DEFAULT_STAGES end + def global_variables + @variables + end + + def job_variables(name) + job = @jobs[name.to_sym] + return [] unless job + + job[:variables] || [] + end + private def initial_parsing - @before_script = @config[:before_script] || [] + @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @stages = @config[:stages] || @config[:types] @variables = @config[:variables] || {} @cache = @config[:cache] + @jobs = {} + @config.except!(*ALLOWED_YAML_KEYS) + @config.each { |name, param| add_job(name, param) } - # anything that doesn't have script is considered as unknown - @config.each do |name, param| - raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) && param.has_key?(:script) - end + raise ValidationError, "Please define at least one job" if @jobs.none? + end - unless @config.values.any?{|job| job.is_a?(Hash)} - raise ValidationError, "Please define at least one job" - end + def add_job(name, job) + return if name.to_s.start_with?('.') - @jobs = {} - @config.each do |key, job| - next if key.to_s.start_with?('.') - stage = job[:stage] || job[:type] || DEFAULT_STAGE - @jobs[key] = { stage: stage }.merge(job) - end + raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script) + + stage = job[:stage] || job[:type] || DEFAULT_STAGE + @jobs[name] = { stage: stage }.merge(job) end def build_job(name, job) { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', + environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, artifacts: job[:artifacts], cache: job[:cache] || @cache, dependencies: job[:dependencies], + after_script: job[:after_script] || @after_script, }.compact } end - def normalize_script(script) - if script.is_a? Array - script.join("\n") - else - script + def validate! + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + + validate_global! + + @jobs.each do |name, job| + validate_job!(name, job) end + + true end - def validate! - unless validate_array_of_strings(@before_script) - raise ValidationError, "before_script should be an array of strings" + def validate_global! + unless @after_script.nil? || validate_array_of_strings(@after_script) + raise ValidationError, "after_script should be an array of strings" end unless @image.nil? || @image.is_a?(String) @@ -115,43 +138,45 @@ module Ci end unless @variables.nil? || validate_variables(@variables) - raise ValidationError, "variables should be a map of key-valued strings" + raise ValidationError, "variables should be a map of key-value strings" end - if @cache - if @cache[:key] && !validate_string(@cache[:key]) - raise ValidationError, "cache:key parameter should be a string" - end + validate_global_cache! if @cache + end - if @cache[:untracked] && !validate_boolean(@cache[:untracked]) - raise ValidationError, "cache:untracked parameter should be an boolean" + def validate_global_cache! + @cache.keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} cache unknown parameter #{key}" end + end - if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) - raise ValidationError, "cache:paths parameter should be an array of strings" - end + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" end - @jobs.each do |name, job| - validate_job!(name, job) + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" end - true + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end end def validate_job!(name, job) validate_job_name!(name) validate_job_keys!(name, job) validate_job_types!(name, job) + validate_job_script!(name, job) validate_job_stage!(name, job) if job[:stage] + validate_job_variables!(name, job) if job[:variables] validate_job_cache!(name, job) if job[:cache] validate_job_artifacts!(name, job) if job[:artifacts] validate_job_dependencies!(name, job) if job[:dependencies] end - private - def validate_job_name!(name) if name.blank? || !validate_string(name) raise ValidationError, "job name should be non-empty string" @@ -167,10 +192,6 @@ module Ci end def validate_job_types!(name, job) - if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name} job: script should be a string or an array of a strings" - end - if job[:image] && !validate_string(job[:image]) raise ValidationError, "#{name} job: image should be a string" end @@ -183,21 +204,39 @@ module Ci raise ValidationError, "#{name} job: tags parameter should be an array of strings" end - if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings" + if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) + raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" end - if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings" + if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) + raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" end if job[:allow_failure] && !validate_boolean(job[:allow_failure]) raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + if job[:when] && !job[:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end + + if job[:environment] && !validate_environment(job[:environment]) + raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" + end + end + + def validate_job_script!(name, job) + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" + end + + if job[:before_script] && !validate_array_of_strings(job[:before_script]) + raise ValidationError, "#{name} job: before_script should be an array of strings" + end + + if job[:after_script] && !validate_array_of_strings(job[:after_script]) + raise ValidationError, "#{name} job: after_script should be an array of strings" + end end def validate_job_stage!(name, job) @@ -206,7 +245,20 @@ module Ci end end + def validate_job_variables!(name, job) + unless validate_variables(job[:variables]) + raise ValidationError, + "#{name} job: variables should be a map of key-value strings" + end + end + def validate_job_cache!(name, job) + job[:cache].keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} job: cache unknown parameter #{key}" + end + end + if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" end @@ -221,6 +273,12 @@ module Ci end def validate_job_artifacts!(name, job) + job[:artifacts].keys.each do |key| + unless ALLOWED_ARTIFACTS_KEYS.include? key + raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" + end + end + if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) raise ValidationError, "#{name} job: artifacts:name parameter should be a string" end @@ -232,63 +290,56 @@ module Ci if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end + + if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) + raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" + end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) - if !validate_array_of_strings(job[:dependencies]) + unless validate_array_of_strings(job[:dependencies]) raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end stage_index = stages.index(job[:stage]) job[:dependencies].each do |dependency| - raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency] + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless stages.index(@jobs[dependency][:stage]) < stage_index + unless stages.index(@jobs[dependency.to_sym][:stage]) < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end end - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_boolean(value) - value.in?([true, false]) - end - - def process?(only_params, except_params, ref, tag) + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? - return false unless matching?(only_params, ref, tag) + return false unless matching?(only_params, ref, tag, trigger_request) end if except_params.present? - return false if matching?(except_params, ref, tag) + return false if matching?(except_params, ref, tag, trigger_request) end true end - def matching?(patterns, ref, tag) + def matching?(patterns, ref, tag, trigger_request) patterns.any? do |pattern| - match_ref?(pattern, ref, tag) + match_ref?(pattern, ref, tag, trigger_request) end end - def match_ref?(pattern, ref, tag) + def match_ref?(pattern, ref, tag, trigger_request) pattern, path = pattern.split('@', 2) return false if path && path != self.path return true if tag && pattern == 'tags' return true if !tag && pattern == 'branches' + return true if trigger_request.present? && pattern == 'triggers' if pattern.first == "/" && pattern.last == "/" Regexp.new(pattern[1...-1]) =~ ref diff --git a/lib/ci/status.rb b/lib/ci/status.rb deleted file mode 100644 index 3fb1fe29494..00000000000 --- a/lib/ci/status.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Ci - class Status - def self.get_status(statuses) - if statuses.none? - 'skipped' - elsif statuses.all? { |status| status.success? || status.ignored? } - 'success' - elsif statuses.all?(&:pending?) - 'pending' - elsif statuses.any?(&:running?) || statuses.any?(&:pending?) - 'running' - elsif statuses.all?(&:canceled?) - 'canceled' - else - 'failed' - end - end - end -end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..eb5a2596177 --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + delegate :registry, :client, to: :repository + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] || config['blobSum'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..42232b7129d --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,68 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + response_body @faraday.get("/v2/#{name}/tags/list") + end + + def repository_manifest(name, reference) + response_body @faraday.get("/v2/#{name}/manifests/#{reference}") + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: 'application/json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + + def response_body(response) + response.body if response.success? + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..589f9f4380a --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,16 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..0e634f6b6ef --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,21 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @uri = uri + @path = options[:path] || default_path + @client = ContainerRegistry::Client.new(uri, options) + end + + def repository(name) + ContainerRegistry::Repository.new(self, name) + end + + private + + def default_path + @uri.sub(/^https?:\/\//, '') + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..0e4a7cb3cc9 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + delegate :client, to: :registry + + def initialize(registry, name) + @registry, @name = registry, name + end + + def path + [registry.path, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..7a0929d774e --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,87 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + delegate :registry, :client, to: :repository + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def v1? + manifest && manifest['schemaVersion'] == 1 + end + + def v2? + manifest && manifest['schemaVersion'] == 2 + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + + manifest[key] + end + + def digest + return @digest if defined?(@digest) + + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + + @config_blob = repository.blob(manifest['config']) + end + + def config + return unless config_blob + + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + + layers = manifest['layers'] || manifest['fsLayers'] + + @layers = layers.map do |layer| + repository.blob(layer) + end + end + + def total_size + return unless layers + + layers.map(&:size).sum if v2? + end + + def delete + return unless digest + + client.delete_repository_tag(repository.name, digest) + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f15b2cfd231..668d2fa41b3 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -27,7 +27,7 @@ class EventFilter @params = if params params.dup else - []#EventFilter.default_filter + [] # EventFilter.default_filter end end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 2eae55e534b..440dd44ece7 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -1,9 +1,9 @@ class FileSizeValidator < ActiveModel::EachValidator - MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze - CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze + MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze + CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze - DEFAULT_TOKENIZER = lambda { |value| value.split(//) } - RESERVED_OPTIONS = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long] + DEFAULT_TOKENIZER = -> (value) { value.split(//) }.freeze + RESERVED_OPTIONS = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long].freeze def initialize(options) if range = (options.delete(:in) || options.delete(:within)) diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 6108697bc20..37f4c34054f 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,7 @@ -require 'gitlab/git' +require_dependency 'gitlab/git' module Gitlab + def self.com? + Gitlab.config.gitlab.url == 'https://gitlab.com' + end end diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb index b366c89889e..04676fdb748 100644 --- a/lib/gitlab/akismet_helper.rb +++ b/lib/gitlab/akismet_helper.rb @@ -9,14 +9,22 @@ module Gitlab Gitlab.config.gitlab.url) end + def client_ip(env) + env['action_dispatch.remote_ip'].to_s + end + + def user_agent(env) + env['HTTP_USER_AGENT'] + end + def check_for_spam?(project, user) akismet_enabled? && !project.team.member?(user) end def is_spam?(environment, user, text) client = akismet_client - ip_address = environment['REMOTE_ADDR'] - user_agent = environment['HTTP_USER_AGENT'] + ip_address = client_ip(environment) + user_agent = user_agent(environment) params = { type: 'comment', diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 30509528b8b..db1704af75e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,17 +1,86 @@ module Gitlab - class Auth - def find(login, password) - user = User.by_login(login) - - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? - - Gitlab::LDAP::Authentication.login(login, password) - else - user if user.valid_password?(password) + module Auth + Result = Struct.new(:user, :type) + + class << self + def find_for_git_client(login, password, project:, ip:) + raise "Must provide an IP for rate limiting" if ip.nil? + + result = Result.new + + if valid_ci_request?(login, password, project) + result.type = :ci + elsif result.user = find_with_user_password(login, password) + result.type = :gitlab_or_ldap + elsif result.user = oauth_access_token_check(login, password) + result.type = :oauth + end + + rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) + result + end + + def find_with_user_password(login, password) + user = User.by_login(login) + + # If no user is found, or it's an LDAP server, try LDAP. + # LDAP users are only authenticated via LDAP + if user.nil? || user.ldap_user? + # Second chance - try LDAP authentication + return nil unless Gitlab::LDAP::Config.enabled? + + Gitlab::LDAP::Authentication.login(login, password) + else + user if user.valid_password?(password) + end + end + + def rate_limit!(ip, success:, login:) + rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) + return unless rate_limiter.enabled? + + if success + # Repeated login 'failures' are normal behavior for some Git clients so + # it is important to reset the ban counter once the client has proven + # they are not a 'bad guy'. + rate_limiter.reset! + else + # Register a login failure so that Rack::Attack can block the next + # request from this IP if needed. + rate_limiter.register_fail! + + if rate_limiter.banned? + Rails.logger.info "IP #{ip} failed to login " \ + "as #{login} but has been temporarily banned from Git auth" + end + end + end + + private + + def valid_ci_request?(login, password, project) + matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) + + return false unless project && matched_login.present? + + underscored_service = matched_login['service'].underscore + + if underscored_service == 'gitlab_ci' + project && project.valid_build_token?(password) + elsif Service.available_services_names.include?(underscored_service) + # We treat underscored_service as a trusted input because it is included + # in the Service.available_services_names whitelist. + service = project.public_send("#{underscored_service}_service") + + service && service.activated? && service.valid_token?(password) + end + end + + def oauth_access_token_check(login, password) + if login == "oauth2" && password.present? + token = Doorkeeper::AccessToken.by_token(password) + token && token.accessible? && User.find_by(id: token.resource_owner_id) + end end end end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb new file mode 100644 index 00000000000..1089bc9f89e --- /dev/null +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -0,0 +1,42 @@ +module Gitlab + module Auth + class IpRateLimiter + attr_reader :ip + + def initialize(ip) + @ip = ip + @banned = false + end + + def enabled? + config.enabled + end + + def reset! + Rack::Attack::Allow2Ban.reset(ip, config) + end + + def register_fail! + # Allow2Ban.filter will return false if this IP has not failed too often yet + @banned = Rack::Attack::Allow2Ban.filter(ip, config) do + # If we return false here, the failure for this IP is ignored by Allow2Ban + ip_can_be_banned? + end + end + + def banned? + @banned + end + + private + + def config + Gitlab.config.rack_attack.git_basic_auth + end + + def ip_can_be_banned? + config.ip_whitelist.exclude?(ip) + end + end + end +end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb new file mode 100644 index 00000000000..51b1df9ecbd --- /dev/null +++ b/lib/gitlab/award_emoji.rb @@ -0,0 +1,84 @@ +module Gitlab + class AwardEmoji + CATEGORIES = { + other: "Other", + objects: "Objects", + places: "Places", + travel_places: "Travel", + emoticons: "Emoticons", + objects_symbols: "Symbols", + nature: "Nature", + celebration: "Celebration", + people: "People", + activity: "Activity", + flags: "Flags", + food_drink: "Food" + }.with_indifferent_access + + CATEGORY_ALIASES = { + symbols: "objects_symbols", + foods: "food_drink", + travel: "travel_places" + }.with_indifferent_access + + def self.normalize_emoji_name(name) + aliases[name] || name + end + + def self.emoji_by_category + unless @emoji_by_category + @emoji_by_category = Hash.new { |h, key| h[key] = [] } + + emojis.each do |emoji_name, data| + data["name"] = emoji_name + + # Skip Fitzpatrick(tone) modifiers + next if data["category"] == "modifier" + + category = CATEGORY_ALIASES[data["category"]] || data["category"] + + @emoji_by_category[category] << data + end + + @emoji_by_category = @emoji_by_category.sort.to_h + end + + @emoji_by_category + end + + def self.emojis + @emojis ||= + begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) + JSON.parse(File.read(json_path)) + end + end + + def self.aliases + @aliases ||= + begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) + JSON.parse(File.read(json_path)) + end + end + + # Returns an Array of Emoji names and their asset URLs. + def self.urls + @urls ||= begin + path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + prefix = Gitlab::Application.config.assets.prefix + digest = Gitlab::Application.config.assets.digest + + JSON.parse(File.read(path)).map do |hash| + if digest + fname = "#{hash['unicode']}-#{hash['digest']}" + else + fname = hash['unicode'] + end + + { name: hash['name'], path: "#{prefix}/#{fname}.png" } + end + end + end + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index cdcaae8094c..7e3f5abba62 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,5 +1,3 @@ -require_relative 'shell_env' - module Grack class AuthSpawner def self.call(env) @@ -36,10 +34,7 @@ module Grack lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call return lfs_response unless lfs_response.nil? - if project && authorized_request? - # Tell gitlab-workhorse the request is OK, and what the GL_ID is - render_grack_auth_ok - elsif @user.nil? && !@ci + if @user.nil? && !@ci unauthorized else render_not_found @@ -64,11 +59,6 @@ module Grack end @user = authenticate_user(login, password) - - if @user - Gitlab::ShellEnv.set_env(@user) - @env['REMOTE_USER'] = @auth.username - end end def ci_request?(login, password) @@ -98,7 +88,7 @@ module Grack end def authenticate_user(login, password) - user = Gitlab::Auth.new.find(login, password) + user = Gitlab::Auth.find_with_user_password(login, password) unless user user = oauth_access_token_check(login, password) @@ -141,36 +131,6 @@ module Grack user end - def authorized_request? - return true if @ci - - case git_cmd - when *Gitlab::GitAccess::DOWNLOAD_COMMANDS - if !Gitlab.config.gitlab_shell.upload_pack - false - elsif user - Gitlab::GitAccess.new(user, project).download_access_check.allowed? - elsif project.public? - # Allow clone/fetch for public projects - true - else - false - end - when *Gitlab::GitAccess::PUSH_COMMANDS - if !Gitlab.config.gitlab_shell.receive_pack - false - elsif user - # Skip user authorization on upload request. - # It will be done by the pre-receive hook in the repository. - true - else - false - end - else - false - end - end - def git_cmd if @request.get? @request.params['service'] @@ -197,24 +157,6 @@ module Grack end end - def render_grack_auth_ok - repo_path = - if @request.path_info =~ /^([\w\.\/-]+)\.wiki\.git/ - ProjectWiki.new(project).repository.path_to_repo - else - project.repository.path_to_repo - end - - [ - 200, - { "Content-Type" => "application/json" }, - [JSON.dump({ - 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), - 'RepoPath' => repo_path, - })] - ] - end - def render_not_found [404, { "Content-Type" => "text/plain" }, ["Not Found"]] end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index b9bb6e76081..3e3986d6382 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -54,19 +54,6 @@ module Gitlab "#{path}.git", "#{new_path}.git"]) end - # Update HEAD for repository - # - # path - project path with namespace - # branch - repository branch name - # - # Ex. - # update_repository_head("gitlab/gitlab-ci", "3-1-stable") - # - def update_repository_head(path, branch) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'update-head', - "#{path}.git", branch]) - end - # Fork repository to new namespace # # path - project path with namespace @@ -92,64 +79,6 @@ module Gitlab 'rm-project', "#{name}.git"]) end - # Add repository branch from passed ref - # - # path - project path with namespace - # branch_name - new branch name - # ref - HEAD for new branch - # - # Ex. - # add_branch("gitlab/gitlab-ci", "4-0-stable", "master") - # - def add_branch(path, branch_name, ref) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'create-branch', - "#{path}.git", branch_name, ref]) - end - - # Remove repository branch - # - # path - project path with namespace - # branch_name - branch name to remove - # - # Ex. - # rm_branch("gitlab/gitlab-ci", "4-0-stable") - # - def rm_branch(path, branch_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-branch', - "#{path}.git", branch_name]) - end - - # Add repository tag from passed ref - # - # path - project path with namespace - # tag_name - new tag name - # ref - HEAD for new tag - # message - optional message for tag (annotated tag) - # - # Ex. - # add_tag("gitlab/gitlab-ci", "v4.0", "master") - # add_tag("gitlab/gitlab-ci", "v4.0", "master", "message") - # - def add_tag(path, tag_name, ref, message = nil) - cmd = %W(#{gitlab_shell_path}/bin/gitlab-projects create-tag #{path}.git - #{tag_name} #{ref}) - cmd << message unless message.nil? || message.empty? - Gitlab::Utils.system_silent(cmd) - end - - # Remove repository tag - # - # path - project path with namespace - # tag_name - tag name to remove - # - # Ex. - # rm_tag("gitlab/gitlab-ci", "v4.0") - # - def rm_tag(path, tag_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-tag', - "#{path}.git", tag_name]) - end - # Gc repository # # path - project path with namespace @@ -251,7 +180,7 @@ module Gitlab # exists?('gitlab/cookies.git') # def exists?(dir_name) - File.exists?(full_path(dir_name)) + File.exist?(full_path(dir_name)) end protected diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb deleted file mode 100644 index 9f5adee594a..00000000000 --- a/lib/gitlab/backend/shell_env.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - # This module provide 2 methods - # to set specific ENV variables for GitLab Shell - module ShellEnv - extend self - - def set_env(user) - # Set GL_ID env variable - if user - ENV['GL_ID'] = gl_id(user) - end - end - - def reset_env - # Reset GL_ID env variable - ENV['GL_ID'] = nil - end - - def gl_id(user) - if user.present? - "user-#{user.id}" - else - # This empty string is used in the render_grack_auth_ok method - "" - end - end - end -end diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb new file mode 100644 index 00000000000..e5e9fab3f5c --- /dev/null +++ b/lib/gitlab/badge/build.rb @@ -0,0 +1,46 @@ +module Gitlab + module Badge + ## + # Build badge + # + class Build + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(project, ref) + @project, @ref = project, ref + @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) + end + + def type + 'image/svg+xml' + end + + def data + File.read(@image[:path]) + end + + def to_s + @image[:name].sub(/\.svg$/, '') + end + + def to_html + link_to(image_tag(image_url, alt: 'build status'), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index d88a6eaac6b..8d1ad62fae0 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -5,6 +5,17 @@ module Gitlab attr_reader :consumer, :api + def self.from_project(project) + import_data_credentials = project.import_data.credentials if project.import_data + if import_data_credentials && import_data_credentials[:bb_session] + token = import_data_credentials[:bb_session][:bitbucket_access_token] + token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] + new(token, token_secret) + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" + end + end + def initialize(access_token = nil, access_token_secret = nil) @consumer = ::OAuth::Consumer.new( config.app_id, @@ -54,7 +65,7 @@ module Gitlab def issues(project_identifier) all_issues = [] offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket + per_page = 50 # Maximum number allowed by Bitbucket index = 0 begin @@ -110,7 +121,7 @@ module Gitlab def get(url) response = api.get(url) - raise Unauthorized if (400..499).include?(response.code.to_i) + raise Unauthorized if (400..499).cover?(response.code.to_i) response end @@ -120,7 +131,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} + Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } end def bitbucket_options diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 46e51a4bf6d..7beaecd1cf0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -5,10 +5,7 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb index f4dd393ad29..e03c3155b3e 100644 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ b/lib/gitlab/bitbucket_import/key_deleter.rb @@ -6,10 +6,7 @@ module Gitlab def initialize(project) @project = project @current_user = project.creator - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) end def execute diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 03aac1a025a..b90ef0b0fba 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["slug"], @@ -21,10 +21,8 @@ module Gitlab import_type: "bitbucket", import_source: "#{repo["owner"]}/#{repo["slug"]}", import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", + import_data: { credentials: { bb_session: session_data } } ).execute - - project.create_import_data(data: { "bb_session" => session_data } ) - project end end end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 34e949130da..9f45aefda0f 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -3,7 +3,7 @@ module Gitlab class << self def build(build) project = build.project - commit = build.commit + commit = build.pipeline user = build.user data = { diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index f2020c82d40..cd2e83b4c27 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab child_pattern = '[^/]*/?$' unless @opts[:recursive] match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ - until gz.eof? do + until gz.eof? begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb new file mode 100644 index 00000000000..b48d3592f16 --- /dev/null +++ b/lib/gitlab/ci/config.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + ## + # Base GitLab CI Configuration facade + # + class Config + delegate :valid?, :errors, to: :@global + + ## + # Temporary delegations that should be removed after refactoring + # + delegate :before_script, to: :@global + + def initialize(config) + @config = Loader.new(config).load! + + @global = Node::Global.new(@config) + @global.process! + end + + def to_hash + @config + end + end + end +end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb new file mode 100644 index 00000000000..dbf6eb0edbe --- /dev/null +++ b/lib/gitlab/ci/config/loader.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + class Config + class Loader + class FormatError < StandardError; end + + def initialize(config) + @config = YAML.safe_load(config, [Symbol], [], true) + end + + def valid? + @config.is_a?(Hash) + end + + def load! + unless valid? + raise FormatError, 'Invalid configuration format' + end + + @config.deep_symbolize_keys + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb new file mode 100644 index 00000000000..d60f87f3f94 --- /dev/null +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -0,0 +1,61 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + def allowed_nodes + self.class.allowed_nodes || {} + end + + private + + def prevalidate! + unless @value.is_a?(Hash) + @errors << 'should be a configuration entry with hash value' + end + end + + def create_node(key, factory) + factory.with(value: @value[key]) + factory.nullify! unless @value.has_key?(key) + factory.create! + end + + class_methods do + def allowed_nodes + Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + end + + private + + def allow_node(symbol, entry_class, metadata) + factory = Node::Factory.new(entry_class) + .with(description: metadata[:description]) + + define_method(symbol) do + raise Entry::InvalidError unless valid? + + @nodes[symbol].try(:value) + end + + (@allowed_nodes ||= {}).merge!(symbol => factory) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb new file mode 100644 index 00000000000..52758a962f3 --- /dev/null +++ b/lib/gitlab/ci/config/node/entry.rb @@ -0,0 +1,77 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Base abstract class for each configuration entry node. + # + class Entry + class InvalidError < StandardError; end + + attr_accessor :description + + def initialize(value) + @value = value + @nodes = {} + @errors = [] + + prevalidate! + end + + def process! + return if leaf? + return unless valid? + + compose! + + nodes.each(&:process!) + nodes.each(&:validate!) + end + + def nodes + @nodes.values + end + + def valid? + errors.none? + end + + def leaf? + allowed_nodes.none? + end + + def errors + @errors + nodes.map(&:errors).flatten + end + + def allowed_nodes + {} + end + + def validate! + raise NotImplementedError + end + + def value + raise NotImplementedError + end + + private + + def prevalidate! + end + + def compose! + allowed_nodes.each do |key, essence| + @nodes[key] = create_node(key, essence) + end + end + + def create_node(key, essence) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb new file mode 100644 index 00000000000..787ca006f5a --- /dev/null +++ b/lib/gitlab/ci/config/node/factory.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Factory class responsible for fabricating node entry objects. + # + # It uses Fluent Interface pattern to set all necessary attributes. + # + class Factory + class InvalidFactory < StandardError; end + + def initialize(entry_class) + @entry_class = entry_class + @attributes = {} + end + + def with(attributes) + @attributes.merge!(attributes) + self + end + + def nullify! + @entry_class = Node::Null + self + end + + def create! + raise InvalidFactory unless @attributes.has_key?(:value) + + @entry_class.new(@attributes[:value]).tap do |entry| + entry.description = @attributes[:description] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb new file mode 100644 index 00000000000..044603423d5 --- /dev/null +++ b/lib/gitlab/ci/config/node/global.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a global entry - root node for entire + # GitLab CI Configuration file. + # + class Global < Entry + include Configurable + + allow_node :before_script, Script, + description: 'Script that will be executed before each job.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb new file mode 100644 index 00000000000..4f590f6bec8 --- /dev/null +++ b/lib/gitlab/ci/config/node/null.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a configuration entry that is not being used + # in configuration file. + # + # This implements Null Object pattern. + # + class Null < Entry + def value + nil + end + + def validate! + nil + end + + def method_missing(*) + nil + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb new file mode 100644 index 00000000000..5072bf0db7d --- /dev/null +++ b/lib/gitlab/ci/config/node/script.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a script. + # + # Each element in the value array is a command that will be executed + # by GitLab Runner. Currently we concatenate these commands with + # new line character as a separator, what is compatible with + # implementation in Runner. + # + class Script < Entry + include ValidationHelpers + + def value + @value.join("\n") + end + + def validate! + unless validate_array_of_strings(@value) + @errors << 'before_script should be an array of strings' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb new file mode 100644 index 00000000000..72f648975dc --- /dev/null +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -0,0 +1,55 @@ +module Gitlab + module Ci + class Config + module Node + module ValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + Regexp.new(value[1...-1]) + else + true + end + rescue RegexpError + false + end + + def validate_environment(value) + value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 85583dce9ee..9dc2602867e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -19,7 +19,7 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..(Date.today + 1.day)).to_a + dates = (1.year.ago.to_date..Date.today).to_a dates.each do |date| date_id = date.to_time.to_i.to_s diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 761b63e98f6..28c34429c1f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,18 +1,22 @@ module Gitlab module CurrentSettings def current_application_settings - key = :current_application_settings - - RequestStore.store[key] ||= begin - settings = nil + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - if connect_to_db? - settings = ::ApplicationSetting.current - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? - end + def ensure_application_settings! + settings = ::ApplicationSetting.cached - settings || fake_application_settings + if !settings && connect_to_db? + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end def fake_application_settings @@ -21,21 +25,25 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], - twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], - sign_in_text: Settings.extra['sign_in_text'], + sign_in_text: nil, + after_sign_up_text: nil, + help_page_text: nil, + shared_runners_text: nil, restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, two_factor_grace_period: 48, - akismet_enabled: false + akismet_enabled: false, + repository_checks_enabled: true, + container_registry_token_expire_delay: 5, ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 6f9da69983a..d76ecb54017 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -5,17 +5,35 @@ module Gitlab end def self.mysql? - adapter_name.downcase == 'mysql2' + adapter_name.casecmp('mysql2').zero? end def self.postgresql? - adapter_name.downcase == 'postgresql' + adapter_name.casecmp('postgresql').zero? end def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end + def self.nulls_last_order(field, direction = 'ASC') + order = "#{field} #{direction}" + + if Gitlab::Database.postgresql? + order << ' NULLS LAST' + else + # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL + # columns. In the (default) ascending order, `0` comes first. + order.prepend("#{field} IS NULL, ") if direction == 'ASC' + end + + order + end + + def self.random + Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + end + def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb new file mode 100644 index 00000000000..dec20d8659b --- /dev/null +++ b/lib/gitlab/database/migration_helpers.rb @@ -0,0 +1,158 @@ +module Gitlab + module Database + module MigrationHelpers + # Creates a new index, concurrently when supported + # + # On PostgreSQL this method creates an index concurrently, on MySQL this + # creates a regular index. + # + # Example: + # + # add_concurrent_index :users, :some_column + # + # See Rails' `add_index` for more info on the available arguments. + def add_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'add_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + end + + add_index(table_name, column_name, options) + end + + # Updates the value of a column in batches. + # + # This method updates the table in batches of 5% of the total row count. + # This method will continue updating rows until no rows remain. + # + # When given a block this method will yield two values to the block: + # + # 1. An instance of `Arel::Table` for the table that is being updated. + # 2. The query to run as an Arel object. + # + # By supplying a block one can add extra conditions to the queries being + # executed. Note that the same block is used for _all_ queries. + # + # Example: + # + # update_column_in_batches(:projects, :foo, 10) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # + # This would result in this method updating only rows where + # `projects.some_column` equals "hello". + # + # table - The name of the table. + # column - The name of the column to update. + # value - The value for the column. + # + # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop + # determines this method to be too complex while there's no way to make it + # less "complex" without introducing extra methods (which actually will + # make things _more_ complex). + # + # rubocop: disable Metrics/AbcSize + def update_column_in_batches(table, column, value) + table = Arel::Table.new(table) + + count_arel = table.project(Arel.star.count.as('count')) + count_arel = yield table, count_arel if block_given? + + total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + + return if total == 0 + + # Update in batches of 5% until we run out of any rows to update. + batch_size = ((total / 100.0) * 5.0).ceil + + start_arel = table.project(table[:id]).order(table[:id].asc).take(1) + start_arel = yield table, start_arel if block_given? + start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i + + loop do + stop_arel = table.project(table[:id]). + where(table[:id].gteq(start_id)). + order(table[:id].asc). + take(1). + skip(batch_size) + + stop_arel = yield table, stop_arel if block_given? + stop_row = exec_query(stop_arel.to_sql).to_hash.first + + update_arel = Arel::UpdateManager.new(ActiveRecord::Base). + table(table). + set([[table[column], value]]). + where(table[:id].gteq(start_id)) + + if stop_row + stop_id = stop_row['id'].to_i + start_id = stop_id + update_arel = update_arel.where(table[:id].lt(stop_id)) + end + + update_arel = yield table, update_arel if block_given? + + execute(update_arel.to_sql) + + # There are no more rows left to update. + break unless stop_row + end + end + + # Adds a column with a default value without locking an entire table. + # + # This method runs the following steps: + # + # 1. Add the column with a default value of NULL. + # 2. Change the default value of the column to the specified value. + # 3. Update all existing rows in batches. + # 4. Set a `NOT NULL` constraint on the column if desired (the default). + # + # These steps ensure a column can be added to a large and commonly used + # table without locking the entire table for the duration of the table + # modification. + # + # table - The name of the table to update. + # column - The name of the column to add. + # type - The column type (e.g. `:integer`). + # default - The default value for the column. + # allow_null - When set to `true` the column will allow NULL values, the + # default is to not allow NULL values. + # + # This method can also take a block which is passed directly to the + # `update_column_in_batches` method. + def add_column_with_default(table, column, type, default:, allow_null: false, &block) + if transaction_open? + raise 'add_column_with_default can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + transaction do + add_column(table, column, type, default: nil) + + # Changing the default before the update ensures any newly inserted + # rows already use the proper default value. + change_column_default(table, column, default) + end + + begin + update_column_in_batches(table, column, default, &block) + + change_column_null(table, column, false) unless allow_null + # We want to rescue _all_ exceptions here, even those that don't inherit + # from StandardError. + rescue Exception => error # rubocop: disable all + remove_column(table, column) + + raise error + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index faa2830c16e..d2e85cabf72 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -24,6 +24,10 @@ module Gitlab @lines ||= parser.parse(raw_diff.each_line).to_a end + def too_large? + diff.too_large? + end + def highlighted_diff_lines Gitlab::Diff::Highlight.new(self).highlight end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index dccb717e95d..87a9b1e23ac 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,6 +1,11 @@ module Gitlab module Diff class InlineDiffMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + } + attr_accessor :raw_line, :rich_line def initialize(raw_line, rich_line = raw_line) @@ -8,7 +13,7 @@ module Gitlab @rich_line = ERB::Util.html_escape(rich_line) end - def mark(line_inline_diffs) + def mark(line_inline_diffs, mode: nil, markdown: false) return rich_line unless line_inline_diffs marker_ranges = [] @@ -20,13 +25,22 @@ module Gitlab end offset = 0 - # Mark each range - marker_ranges.each_with_index do |range, i| - class_names = ["idiff"] - class_names << "left" if i == 0 - class_names << "right" if i == marker_ranges.length - 1 - offset = insert_around_range(rich_line, range, "<span class='#{class_names.join(" ")}'>", "</span>", offset) + # Mark each range + marker_ranges.each_with_index do |range, index| + before_content = + if markdown + "{#{MARKDOWN_SYMBOLS[mode]}" + else + "<span class='#{html_class_names(marker_ranges, mode, index)}'>" + end + after_content = + if markdown + "#{MARKDOWN_SYMBOLS[mode]}}" + else + "</span>" + end + offset = insert_around_range(rich_line, range, before_content, after_content, offset) end rich_line.html_safe @@ -34,6 +48,14 @@ module Gitlab private + def html_class_names(marker_ranges, mode, index) + class_names = ["idiff"] + class_names << "left" if index == 0 + class_names << "right" if index == marker_ranges.length - 1 + class_names << mode if mode + class_names.join(" ") + end + # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index d0815fc7eea..522dd2b9428 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,16 +17,16 @@ module Gitlab Enumerator.new do |yielder| @lines.each do |line| next if filename?(line) - - full_line = line.gsub(/\n/, '') - + + full_line = line.delete("\n") + if line.match(/^@@ -/) type = "match" - + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 #top of file + + next if line_old <= 1 && line_new <= 1 # top of file yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next @@ -39,8 +39,8 @@ module Gitlab yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 end - - + + case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 41f0edcaf7e..e2fee6b9f3e 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -2,22 +2,21 @@ module Gitlab module Email module Message class RepositoryPush - attr_accessor :recipient attr_reader :author_id, :ref, :action - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers + include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author delegate :username, to: :author, prefix: :author - def initialize(notify, project_id, recipient, opts = {}) + def initialize(notify, project_id, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless opts[:author_id] && opts[:ref] && opts[:action] @notify = notify @project_id = project_id - @recipient = recipient @opts = opts.dup @author_id = @opts.delete(:author_id) @@ -38,7 +37,7 @@ module Gitlab end def diffs - @diffs ||= (compare.diffs if compare) + @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) end def diffs_count @@ -49,6 +48,10 @@ module Gitlab @opts[:compare] end + def diff_refs + @opts[:diff_refs] + end + def compare_timeout diffs.overflow? if diffs end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2ca21af5bc8..97ef9851d71 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -45,12 +45,12 @@ module Gitlab note = create_note(reply) unless note.persisted? - message = "The comment could not be created for the following reasons:" + msg = "The comment could not be created for the following reasons:" note.errors.full_messages.each do |error| - message << "\n\n- #{error}" + msg << "\n\n- #{error}" end - raise InvalidNoteError, message + raise InvalidNoteError, msg end end @@ -63,9 +63,24 @@ module Gitlab end def reply_key - reply_key = nil + key_from_to_header || key_from_additional_headers + end + + def key_from_to_header + key = nil message.to.each do |address| - reply_key = Gitlab::IncomingEmail.key_from_address(address) + key = Gitlab::IncomingEmail.key_from_address(address) + break if key + end + + key + end + + def key_from_additional_headers + reply_key = nil + + Array(message.references).each do |message_id| + reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id) break if reply_key end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 6ed36b51f12..3411eb1d9ce 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -65,7 +65,7 @@ module Gitlab (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } + break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } # Headers on the same line break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 2ef50286b1d..ffe49364379 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -15,6 +15,25 @@ module Gitlab # seconds then two overlapping operations may hold a lease for the same # key at the same time. # + # This class has no 'cancel' method. I originally decided against adding + # it because it would add complexity and a false sense of security. The + # complexity: instead of setting '1' we would have to set a UUID, and to + # delete it we would have to execute Lua on the Redis server to only + # delete the key if the value was our own UUID. Otherwise there is a + # chance that when you intend to cancel your lease you actually delete + # someone else's. The false sense of security: you cannot design your + # system to rely too much on the lease being cancelled after use because + # the calling (Ruby) process may crash or be killed. You _cannot_ count + # on begin/ensure blocks to cancel a lease, because the 'ensure' does + # not always run. Think of 'kill -9' from the Unicorn master for + # instance. + # + # If you find that leases are getting in your way, ask yourself: would + # it be enough to lower the lease timeout? Another thing that might be + # appropriate is to only use a lease for bulk/automated operations, and + # to ignore the lease when you get a single 'manual' user request (a + # button click). + # class ExclusiveLease def initialize(key, timeout:) @key, @timeout = key, timeout @@ -24,15 +43,14 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + Gitlab::Redis.with do |redis| + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end end - private + # No #cancel method. See comments above! - def redis - # Maybe someday we want to use a connection pool... - @redis ||= Redis.new(url: Gitlab::RedisConfig.url) - end + private def redis_key "gitlab:exclusive_lease:#{@key}" diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 431d50882fd..2152182b37f 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -26,7 +26,7 @@ module Gitlab def user_map users = {} res = @api.command(:listPeople) - res['people']['person'].each do |user| + [res['people']['person']].flatten.each do |user| users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] } end users diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index db580b5e578..501d5a95547 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -8,17 +8,17 @@ module Gitlab import_data = project.import_data.try(:data) repo_data = import_data['repo'] if import_data - @repo = FogbugzImport::Repository.new(repo_data) - - @known_labels = Set.new + if repo_data + @repo = FogbugzImport::Repository.new(repo_data) + @known_labels = Set.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute return true unless repo.valid? - - data = project.import_data.try(:data) - - client = Gitlab::FogbugzImport::Client.new(token: data['fb_session']['token'], uri: data['fb_session']['uri']) + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @categories = client.categories @@ -30,6 +30,10 @@ module Gitlab private + def fb_session + @import_data_credentials ||= project.import_data.credentials[:fb_session] if project.import_data && project.import_data.credentials + end + def user_map @user_map ||= begin user_map = Hash.new @@ -236,9 +240,8 @@ module Gitlab end def build_attachment_url(rel_url) - data = project.import_data.try(:data) - uri = data['fb_session']['uri'] - token = data['fb_session']['token'] + uri = fb_session[:uri] + token = fb_session[:token] "#{uri}/#{rel_url}&token=#{token}" end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index e0163499e30..1918d5b208d 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.safe_name, path: repo.path, @@ -21,18 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::INTERNAL, import_type: 'fogbugz', import_source: repo.name, - import_url: Project::UNKNOWN_IMPORT_URL + import_url: Project::UNKNOWN_IMPORT_URL, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session } } ).execute - - project.create_import_data( - data: { - 'repo' => repo.raw_data, - 'user_map' => user_map, - 'fb_session' => fb_session - } - ) - - project end end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb new file mode 100644 index 00000000000..78d7a4f27cf --- /dev/null +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -0,0 +1,84 @@ +module Gitlab + module Gfm + ## + # Class that unfolds local references in text. + # + # The initializer takes text in Markdown and project this text is valid + # in context of. + # + # `unfold` method tries to find all local references and unfold each of + # those local references to cross reference format, assuming that the + # argument passed to this method is a project that references will be + # viewed from (see `Referable#to_reference method). + # + # Examples: + # + # 'Hello, this issue is related to #123 and + # other issues labeled with ~"label"', will be converted to: + # + # 'Hello, this issue is related to gitlab-org/gitlab-ce#123 and + # other issue labeled with gitlab-org/gitlab-ce~"label"'. + # + # It does respect markdown lexical rules, so text in code block will not be + # replaced, see another example: + # + # 'Merge request for issue #1234, see also link: + # http://gitlab.com/some/link/#1234, and code `puts #1234`' => + # + # 'Merge request for issue gitlab-org/gitlab-ce#1234, se also link: + # http://gitlab.com/some/link/#1234, and code `puts #1234`' + # + class ReferenceRewriter + def initialize(text, source_project, current_user) + @text = text + @source_project = source_project + @current_user = current_user + @original_html = markdown(text) + @pattern = Gitlab::ReferenceExtractor.references_pattern + end + + def rewrite(target_project) + return @text unless needs_rewrite? + + @text.gsub(@pattern) do |reference| + unfold_reference(reference, Regexp.last_match, target_project) + end + end + + def needs_rewrite? + @text =~ @pattern + end + + private + + def unfold_reference(reference, match, target_project) + before = @text[0...match.begin(0)] + after = @text[match.end(0)..-1] + + referable = find_referable(reference) + return reference unless referable + + cross_reference = referable.to_reference(target_project) + return reference if reference == cross_reference + + new_text = before + cross_reference + after + substitution_valid?(new_text) ? cross_reference : reference + end + + def find_referable(reference) + extractor = Gitlab::ReferenceExtractor.new(@source_project, + @current_user) + extractor.analyze(reference) + extractor.all.first + end + + def substitution_valid?(substituted) + @original_html == markdown(substituted) + end + + def markdown(text) + Banzai.render(text, project: @source_project, no_original_data: true) + end + end + end +end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb new file mode 100644 index 00000000000..abc8c8c55e6 --- /dev/null +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -0,0 +1,51 @@ +module Gitlab + module Gfm + ## + # Class that rewrites markdown links for uploads + # + # Using a pattern defined in `FileUploader` it copies files to a new + # project and rewrites all links to uploads in in a given text. + # + # + class UploadsRewriter + def initialize(text, source_project, _current_user) + @text = text + @source_project = source_project + @pattern = FileUploader::MARKDOWN_PATTERN + end + + def rewrite(target_project) + return @text unless needs_rewrite? + + @text.gsub(@pattern) do |markdown| + file = find_file(@source_project, $~[:secret], $~[:file]) + return markdown unless file.try(:exists?) + + new_uploader = FileUploader.new(target_project) + new_uploader.store!(file) + new_uploader.to_markdown + end + end + + def needs_rewrite? + files.any? + end + + def files + referenced_files = @text.scan(@pattern).map do + find_file(@source_project, $~[:secret], $~[:file]) + end + + referenced_files.compact.select(&:exists?) + end + + private + + def find_file(project, secret, file) + uploader = FileUploader.new(project, secret) + uploader.retrieve_from_store!(file) + uploader.file + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 3ed1eec517c..d2a0e316cbe 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -122,20 +122,25 @@ module Gitlab build_status_object(true) end + def can_user_do_action?(action) + @permission_cache ||= {} + @permission_cache[action] ||= user.can?(action, project) + end + def change_access_check(change) oldrev, newrev, ref = change.split(' ') action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) - elsif protected_tag?(tag_name(ref)) + elsif (tag_ref = tag_name(ref)) && protected_tag?(tag_ref) # Prevent any changes to existing git tag unless user has permissions :admin_project else :push_code end - unless user.can?(action, project) + unless can_user_do_action?(action) status = case action when :force_push_code_to_protected_branches @@ -176,7 +181,7 @@ module Gitlab end def protected_tag?(tag_name) - project.repository.tag_names.include?(tag_name) + project.repository.tag_exists?(tag_name) end def user_allowed? diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 202263c6742..72992baffd4 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -9,6 +9,10 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new end + def create! + self.klass.create!(self.attributes) + end + private def gl_user_id(github_id) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb new file mode 100644 index 00000000000..a15fc84b418 --- /dev/null +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -0,0 +1,29 @@ +module Gitlab + module GithubImport + class BranchFormatter < BaseFormatter + delegate :repo, :sha, :ref, to: :raw_data + + def exists? + project.repository.branch_exists?(ref) + end + + def name + @name ||= exists? ? ref : "#{ref}-#{short_id}" + end + + def valid? + repo.present? + end + + def valid? + repo.present? + end + + private + + def short_id + sha.to_s[0..7] + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 74d1529e1ff..d325eca6d99 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,18 +1,28 @@ module Gitlab module GithubImport class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :client, :api def initialize(access_token) @client = ::OAuth2::Client.new( config.app_id, config.app_secret, - github_options + github_options.merge(ssl: { verify: config['verify_ssl'] }) ) if access_token - ::Octokit.auto_paginate = true - @api = ::Octokit::Client.new(access_token: access_token) + ::Octokit.auto_paginate = false + + @api = ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + connection_options: { + ssl: { verify: config['verify_ssl'] } + } + ) end end @@ -29,7 +39,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - api.send(method, *args, &block) + request { api.send(method, *args, &block) } else super(method, *args, &block) end @@ -42,11 +52,39 @@ module Gitlab private def config - Gitlab.config.omniauth.providers.find{|provider| provider.name == "github"} + Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end def github_options - OmniAuth::Strategies::GitHub.default_options[:client_options].to_h.symbolize_keys + config["args"]["client_options"].deep_symbolize_keys + end + + def rate_limit + api.rate_limit! + end + + def rate_limit_exceed? + rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = yield + + last_response = api.last_response + + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + data.concat(last_response.data) if last_response.data.is_a?(Array) + end + + data end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 7d58e53991a..2c1b94ef2cd 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -8,6 +8,7 @@ module Gitlab commit_id: raw_data.commit_id, line_code: line_code, author_id: author_id, + type: type, created_at: raw_data.created_at, updated_at: raw_data.updated_at } @@ -28,18 +29,35 @@ module Gitlab end def line_code - if on_diff? - Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) - end + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) end def on_diff? - raw_data.path && raw_data.position + diff_hunk.present? + end + + def diff_hunk + raw_data.diff_hunk + end + + def file_path + raw_data.path end def note formatter.author_line(author) + body end + + def type + 'LegacyDiffNote' if on_diff? + end end end end diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb new file mode 100644 index 00000000000..db1fabaa18a --- /dev/null +++ b/lib/gitlab/github_import/hook_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class HookFormatter + EVENTS = %w[* create delete pull_request push].freeze + + attr_reader :raw + + delegate :id, :name, :active, to: :raw + + def initialize(raw) + @raw = raw + end + + def config + raw.config.attrs + end + + def valid? + (EVENTS & raw.events).any? && active + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 172c5441e36..2286ac8829c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,34 +3,60 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :project, :client + attr_reader :client, :project, :repo, :repo_url def initialize(project) - @project = project - import_data = project.import_data.try(:data) - github_session = import_data["github_session"] if import_data - @client = Client.new(github_session["github_access_token"]) - @formatter = Gitlab::ImportFormatter.new + @project = project + @repo = project.import_source + @repo_url = project.import_url + + if credentials + @client = Client.new(credentials[:user]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute - import_issues && import_pull_requests && import_wiki + import_labels && import_milestones && import_issues && + import_pull_requests && import_wiki end private + def credentials + @credentials ||= project.import_data.credentials if project.import_data + end + + def import_labels + labels = client.labels(repo, per_page: 100) + labels.each { |raw| LabelFormatter.new(project, raw).create! } + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + + def import_milestones + milestones = client.milestones(repo, state: :all, per_page: 100) + milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + def import_issues - client.list_issues(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| - gh_issue = IssueFormatter.new(project, raw_data) + issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - if gh_issue.valid? - issue = Issue.create!(gh_issue.attributes) + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw) - if gh_issue.has_comments? - import_comments(gh_issue.number, issue) - end + if gh_issue.valid? + issue = gh_issue.create! + apply_labels(issue) + import_comments(issue) if gh_issue.has_comments? end end @@ -40,40 +66,101 @@ module Gitlab end def import_pull_requests - client.pull_requests(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| - pull_request = PullRequestFormatter.new(project, raw_data) - - if pull_request.valid? - merge_request = MergeRequest.new(pull_request.attributes) - - if merge_request.save - import_comments(pull_request.number, merge_request) - import_comments_on_diff(pull_request.number, merge_request) - end - end + disable_webhooks + + pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) + pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) + + source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } + target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } + branches_removed = source_branches_removed | target_branches_removed + + restore_branches(branches_removed) + + pull_requests.each do |pull_request| + merge_request = pull_request.create! + apply_labels(merge_request) + import_comments(merge_request) + import_comments_on_diff(merge_request) end true rescue ActiveRecord::RecordInvalid => e raise Projects::ImportService::Error, e.message + ensure + clean_up_restored_branches(branches_removed) + clean_up_disabled_webhooks + end + + def disable_webhooks + update_webhooks(hooks, active: false) + end + + def clean_up_disabled_webhooks + update_webhooks(hooks, active: true) + end + + def update_webhooks(hooks, options) + hooks.each do |hook| + client.edit_hook(repo, hook.id, hook.name, hook.config, options) + end + end + + def hooks + @hooks ||= + begin + client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + + # The GitHub Repository Webhooks API returns 404 for users + # without admin access to the repository when listing hooks. + # In this case we just want to return gracefully instead of + # spitting out an error and stop the import process. + rescue Octokit::NotFound + [] + end + end + + def restore_branches(branches) + branches.each do |name, sha| + client.create_ref(repo, "refs/heads/#{name}", sha) + end + + project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') + end + + def clean_up_restored_branches(branches) + branches.each do |name, _| + client.delete_ref(repo, "heads/#{name}") + project.repository.rm_branch(project.creator, name) + end + end + + def apply_labels(issuable) + issue = client.issue(repo, issuable.iid) + + if issue.labels.count > 0 + label_ids = issue.labels.map do |raw| + Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) + end + + issuable.update_attribute(:label_ids, label_ids) + end end - def import_comments(issue_number, noteable) - comments = client.issue_comments(project.import_source, issue_number) - create_comments(comments, noteable) + def import_comments(issuable) + comments = client.issue_comments(repo, issuable.iid, per_page: 100) + create_comments(issuable, comments) end - def import_comments_on_diff(pull_request_number, merge_request) - comments = client.pull_request_comments(project.import_source, pull_request_number) - create_comments(comments, merge_request) + def import_comments_on_diff(merge_request) + comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) + create_comments(merge_request, comments) end - def create_comments(comments, noteable) - comments.each do |raw_data| - comment = CommentFormatter.new(project, raw_data) - noteable.notes.create!(comment.attributes) + def create_comments(issuable, comments) + comments.each do |raw| + comment = CommentFormatter.new(project, raw) + issuable.notes.create!(comment.attributes) end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 1e3ba44f27c..835ec858b35 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -3,7 +3,9 @@ module Gitlab class IssueFormatter < BaseFormatter def attributes { + iid: number, project: project, + milestone: milestone, title: raw_data.title, description: description, state: state, @@ -18,6 +20,10 @@ module Gitlab raw_data.comments > 0 end + def klass + Issue + end + def number raw_data.number end @@ -54,6 +60,12 @@ module Gitlab @formatter.author_line(author) + body end + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end + end + def state raw_data.state == 'closed' ? 'closed' : 'opened' end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb new file mode 100644 index 00000000000..9f18244e7d7 --- /dev/null +++ b/lib/gitlab/github_import/label_formatter.rb @@ -0,0 +1,27 @@ +module Gitlab + module GithubImport + class LabelFormatter < BaseFormatter + def attributes + { + project: project, + title: title, + color: color + } + end + + def klass + Label + end + + private + + def color + "##{raw_data.color}" + end + + def title + raw_data.name + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb new file mode 100644 index 00000000000..53d4b3102d1 --- /dev/null +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -0,0 +1,52 @@ +module Gitlab + module GithubImport + class MilestoneFormatter < BaseFormatter + def attributes + { + iid: number, + project: project, + title: title, + description: description, + due_date: due_date, + state: state, + created_at: created_at, + updated_at: updated_at + } + end + + def klass + Milestone + end + + private + + def number + raw_data.number + end + + def title + raw_data.title + end + + def description + raw_data.description + end + + def due_date + raw_data.due_on + end + + def state + raw_data.state == 'closed' ? 'closed' : 'active' + end + + def created_at + raw_data.created_at + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 474927069a5..f4221003db5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -23,9 +23,6 @@ module Gitlab import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute - - project.create_import_data(data: { "github_session" => session_data } ) - project end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 4e507b090e8..498b00cb658 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,15 +1,22 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter + delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true + delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true + def attributes { + iid: number, title: raw_data.title, description: description, - source_project: source_project, - source_branch: source_branch.name, - target_project: target_project, - target_branch: target_branch.name, + source_project: source_branch_project, + source_branch: source_branch_name, + head_source_sha: source_branch_sha, + target_project: target_branch_project, + target_branch: target_branch_name, + base_target_sha: target_branch_sha, state: state, + milestone: milestone, author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, @@ -17,12 +24,24 @@ module Gitlab } end + def klass + MergeRequest + end + def number raw_data.number end def valid? - !cross_project? && source_branch.present? && target_branch.present? + source_branch.valid? && target_branch.valid? && !cross_project? + end + + def source_branch + @source_branch ||= BranchFormatter.new(project, raw_data.head) + end + + def target_branch + @target_branch ||= BranchFormatter.new(project, raw_data.base) end private @@ -50,42 +69,23 @@ module Gitlab end def cross_project? - source_repo.present? && target_repo.present? && source_repo.id != target_repo.id + source_branch_repo.id != target_branch_repo.id end def description formatter.author_line(author) + body end - def source_project - project - end - - def source_repo - raw_data.head.repo - end - - def source_branch - source_project.repository.find_branch(raw_data.head.ref) - end - - def target_project - project - end - - def target_repo - raw_data.base.repo - end - - def target_branch - target_project.repository.find_branch(raw_data.base.ref) + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end end def state - @state ||= case true - when raw_data.state == 'closed' && raw_data.merged_at.present? + @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? 'merged' - when raw_data.state == 'closed' + elsif raw_data.state == 'closed' 'closed' else 'opened' diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb new file mode 100644 index 00000000000..f46b43b61a4 --- /dev/null +++ b/lib/gitlab/gitignore.rb @@ -0,0 +1,56 @@ +module Gitlab + class Gitignore + FILTER_REGEX = /\.gitignore\z/.freeze + + def initialize(path) + @path = path + end + + def name + File.basename(@path, '.gitignore') + end + + def content + File.read(@path) + end + + class << self + def all + languages_frameworks + global + end + + def find(key) + file_name = "#{key}.gitignore" + + directory = select_directory(file_name) + directory ? new(File.join(directory, file_name)) : nil + end + + def global + files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } + end + + def languages_frameworks + files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } + end + + private + + def select_directory(file_name) + [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } + end + + def global_dir + File.join(gitignore_dir, 'Global') + end + + def gitignore_dir + Rails.root.join('vendor/gitignore') + end + + def files_for_folder(dir) + Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 850b73244c6..3f76ec97977 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,16 +5,19 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - gitlab_session = import_data["gitlab_session"] if import_data - @client = Client.new(gitlab_session["gitlab_access_token"]) - @formatter = Gitlab::ImportFormatter.new + import_data = project.import_data + if import_data && import_data.credentials && import_data.credentials[:password] + @client = Client.new(import_data.credentials[:password]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute project_identifier = CGI.escape(project.import_source) - #Issues && Comments + # Issues && Comments issues = client.issues(project_identifier) issues.each do |issue| diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 7baaadb813c..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,9 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project.create_import_data(data: { "gitlab_session" => session_data } ) - project end end end diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb new file mode 100644 index 00000000000..624fd00367e --- /dev/null +++ b/lib/gitlab/gl_id.rb @@ -0,0 +1,11 @@ +module Gitlab + module GlId + def self.gl_id(user) + if user.present? + "user-#{user.id}" + else + "" + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb new file mode 100644 index 00000000000..f751a3a12fd --- /dev/null +++ b/lib/gitlab/gon_helper.rb @@ -0,0 +1,19 @@ +module Gitlab + module GonHelper + def add_gon_variables + gon.api_version = API::API.version + gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_issues_tracker = Project.new.default_issue_tracker.to_param + gon.max_file_size = current_application_settings.max_attachment_size + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.shortcuts_path = help_shortcuts_path + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.award_menu_url = emojis_path + + if current_user + gon.current_user_id = current_user.id + gon.api_token = current_user.private_token + end + end + end +end diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 87821c23460..326cfcaa8af 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -21,17 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::PUBLIC, import_type: "google_code", import_source: repo.name, - import_url: repo.import_url + import_url: repo.import_url, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } ).execute - - project.create_import_data( - data: { - "repo" => repo.raw_data, - "user_map" => user_map - } - ) - - project end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index cac76442321..280120b0f9e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,8 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true) - new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) + def self.highlight(blob_name, blob_content, nowrap: true, plain: false) + new(blob_name, blob_content, nowrap: nowrap). + highlight(blob_content, continue: false, plain: plain) end def self.highlight_lines(repository, ref, file_name) @@ -17,8 +18,12 @@ module Gitlab @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText end - def highlight(text, continue: true) - @formatter.format(@lexer.lex(text, continue: continue)).html_safe + def highlight(text, continue: true, plain: false) + if plain + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + else + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + end rescue @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb new file mode 100644 index 00000000000..624c1766024 --- /dev/null +++ b/lib/gitlab/import_export.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + extend self + + VERSION = '0.1.0' + + def export_path(relative_path:) + File.join(storage_path, relative_path) + end + + def storage_path + File.join(Settings.shared['path'], 'tmp/project_exports') + end + + def project_filename + "project.json" + end + + def project_bundle_filename + "project.bundle" + end + + def config_file + 'lib/gitlab/import_export/import_export.yml' + end + + def version_filename + 'VERSION' + end + + def version + VERSION + end + + def reset_tokens? + true + end + end +end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..d230de781d5 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,47 @@ +module Gitlab + module ImportExport + class AttributesFinder + + def initialize(included_attributes:, excluded_attributes:, methods:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + @methods = methods || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + + private + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..78664f076eb --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,40 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def git_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) + end + + private + + def tar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir} .)) + end + + def untar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + + def git_bin_path + Gitlab.config.git.bin_path + end + end + end +end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..e341c4d9cf8 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..164ab6238c4 --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,54 @@ +# Model relationships to be included in the project import/export +project_tree: + - issues: + - notes: + :author + - :labels + - :milestones + - snippets: + - notes: + :author + - :releases + - :events + - project_members: + - :user + - merge_requests: + - notes: + :author + - :merge_request_diff + - pipelines: + - notes: + :author + - :statuses + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches + +# Only include the following attributes for the models specified. +included_attributes: + project: + - :description + - :issues_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + user: + - :id + - :email + - :username + author: + - :name + +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: + - :expired_at + +methods: + statuses: + - :type
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..d209e04f7be --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,64 @@ +module Gitlab + module ImportExport + class Importer + + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + end + + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end + end + + private + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..c569a35a48b --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,68 @@ +module Gitlab + module ImportExport + class MembersMapper + + attr_reader :missing_author_ids + + def initialize(exported_members:, user:, project:) + @exported_members = exported_members + @user = user + @project = project + @missing_author_ids = [] + + # This needs to run first, as second call would be from #map + # which means project members already exist. + ensure_default_member! + end + + def map + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def default_user_id + @user.id + end + + private + + def missing_keys_tracking_hash + Hash.new do |_, key| + @missing_author_ids << key + default_user_id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) + member['user'] = existing_user + + ProjectMember.create(member_hash(member)).persisted? + end + + def member_hash(member) + member.except('id').merge(source_id: @project.id, importing: true) + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..89388d1984b --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..dd71b92c522 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,105 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + + def initialize(user:, shared:, project:) + @path = File.join(shared.export_path, 'project.json') + @user = user + @shared = shared + @project = project + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + rescue => e + @shared.error(e) + false + end + + def restored_project + @restored_project ||= restore_project + end + + private + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project: restored_project) + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations + saved = [] + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + saved << restored_project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end + end + + def restore_project + return @project unless @tree_hash + + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + @project.update(project_params) + @project + end + + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash + def create_sub_relations(relation, tree_hash) + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| + relation.values.flatten.each do |sub_relation| + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + end + + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + [relation_hash, sub_relation] + end + + def create_relation(relation, relation_hash_list) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => restored_project.id), + members_mapper: members_mapper, + user: @user) + end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..9153088e966 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(full_path, project_json_tree) + true + rescue => e + @shared.error(e) + false + end + + private + + def project_json_tree + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..19defd8f03a --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,117 @@ +module Gitlab + module ImportExport + class Reader + + attr_reader :tree + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..b872780f20a --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,128 @@ +module Gitlab + module ImportExport + class RelationFactory + + OVERRIDES = { snippets: :project_snippets, + pipelines: 'Ci::Pipeline', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user = user + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + + generate_imported_object + end + + private + + def update_user_references + USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + + # Users with admin access can map users + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id + + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + end + + def missing_author?(old_author_id) + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil + end + else + imported_object + end + end + + def update_project_references + project_id = @relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end + end + + def reset_ci_tokens + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil + end + + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize + end + + def imported_object + imported_object = relation_class.new(@relation_hash) + yield(imported_object) if block_given? + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user.is_admin? + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..546dae4d122 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:, path_to_bundle:, wiki: false) + @project = project + @path_to_bundle = path_to_bundle + @shared = shared + @wiki = wiki + end + + def restore + return wiki? unless File.exist?(@path_to_bundle) + + FileUtils.mkdir_p(path_to_repo) + + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue => e + @shared.error(e) + false + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + + def wiki? + @wiki + end + end + end +end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..cce43fe994b --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..f38229c6c59 --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,42 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + if compress_and_save + remove_export_path + Rails.logger.info("Saved project export #{archive_file}") + archive_file + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @shared.export_path) + end + + def remove_export_path + FileUtils.rm_rf(@shared.export_path) + end + + def archive_file + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..6aff05b886a --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class Shared + + attr_reader :errors, :opts + + def initialize(opts) + @opts = opts + @errors = [] + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + end + + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..df19354b76e --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,14 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + def restore + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..7292e9d9712 --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + copy_files(uploads_path, uploads_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + end + end + end +end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..cf5c62c5e3c --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionChecker + + def self.check!(*args) + new(*args).check! + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..f7f73dc9343 --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class VersionSaver + + def initialize(shared:) + @shared = shared + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..1eedae39f8a --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..948d43582cf 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', + 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 9068d79c95e..8ce9d32abe0 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,13 +1,10 @@ module Gitlab module IncomingEmail class << self - def enabled? - config.enabled && address_formatted_correctly? - end + FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze - def address_formatted_correctly? - config.address && - config.address.include?("%{key}") + def enabled? + config.enabled && config.address end def reply_address(key) @@ -24,6 +21,13 @@ module Gitlab match[1] end + def key_from_fallback_reply_message_id(message_id) + match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX) + return unless match + + match[1] + end + def config Gitlab.config.incoming_email end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index baf52ff750d..8684b4636ea 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -17,9 +17,9 @@ module Gitlab file.rewind cmd = [] - cmd.push *%W(ssh-keygen) - cmd.push *%W(-E md5) if explicit_fingerprint_algorithm? - cmd.push *%W(-lf #{file.path}) + cmd.push('ssh-keygen') + cmd.push('-E', 'md5') if explicit_fingerprint_algorithm? + cmd.push('-lf', file.path) cmd_output, cmd_status = popen(cmd, '/tmp') end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb new file mode 100644 index 00000000000..2a659ae4c74 --- /dev/null +++ b/lib/gitlab/lazy.rb @@ -0,0 +1,34 @@ +module Gitlab + # A class that can be wrapped around an expensive method call so it's only + # executed when actually needed. + # + # Usage: + # + # object = Gitlab::Lazy.new { some_expensive_work_here } + # + # object['foo'] + # object.bar + class Lazy < BasicObject + def initialize(&block) + @block = block + end + + def method_missing(name, *args, &block) + __evaluate__ + + @result.__send__(name, *args, &block) + end + + def respond_to_missing?(name, include_private = false) + __evaluate__ + + @result.respond_to?(name, include_private) || super + end + + private + + def __evaluate__ + @result = @block.call unless defined?(@result) + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index da4435c7308..f2b649e50a2 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -33,7 +33,10 @@ module Gitlab def allowed? if ldap_user - return true unless ldap_config.active_directory + unless ldap_config.active_directory + user.activate if user.ldap_blocked? + return true + end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index aff7ccb157f..f9bb5775323 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -93,6 +93,7 @@ module Gitlab end protected + def base_config Gitlab.config.ldap end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index a5f767b134d..dda371e6554 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -40,7 +40,7 @@ module Gitlab # Returns boolean def plain?(filename) filename.downcase.end_with?('.txt') || - filename.downcase == 'readme' + filename.casecmp('readme').zero? end def previewable?(filename) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 88a265c6af2..49f702f91f6 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -14,7 +14,8 @@ module Gitlab method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15 + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 } end @@ -41,9 +42,9 @@ module Gitlab prepared = prepare_metrics(metrics) pool.with do |connection| - prepared.each do |metric| + prepared.each_slice(settings[:packet_size]) do |slice| begin - connection.write_points([metric]) + connection.write_points(slice) rescue StandardError end end @@ -70,6 +71,59 @@ module Gitlab value.to_s.gsub('=', '\\=') end + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def self.measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def self.tag_transaction(name, value) + trans = current_transaction + + trans.add_tag(name, value) if trans + end + + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def self.action=(action) + trans = current_transaction + + trans.action = action if trans + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -81,5 +135,11 @@ module Gitlab new(udp: { host: host, port: port }) end end + + private + + def self.current_transaction + Transaction.current + end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index face1921d2e..dcec7543c13 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -11,6 +11,8 @@ module Gitlab module Instrumentation SERIES = 'method_calls' + PROXY_IVAR = :@__gitlab_instrumentation_proxy + def self.configure yield self end @@ -54,7 +56,7 @@ module Gitlab end end - # Instruments all public methods of a module. + # Instruments all public and private methods of a module. # # This method optionally takes a block that can be used to determine if a # method should be instrumented or not. The block is passed the receiving @@ -63,7 +65,8 @@ module Gitlab # # mod - The module to instrument. def self.instrument_methods(mod) - mod.public_methods(false).each do |name| + methods = mod.methods(false) + mod.private_methods(false) + methods.each do |name| method = mod.method(name) if method.owner == mod.singleton_class @@ -74,13 +77,14 @@ module Gitlab end end - # Instruments all public instance methods of a module. + # Instruments all public and private instance methods of a module. # # See `instrument_methods` for more information. # # mod - The module to instrument. def self.instrument_instance_methods(mod) - mod.public_instance_methods(false).each do |name| + methods = mod.instance_methods(false) + mod.private_instance_methods(false) + methods.each do |name| method = mod.instance_method(name) if method.owner == mod @@ -91,6 +95,18 @@ module Gitlab end end + # Returns true if a module is instrumented. + # + # mod - The module to check + def self.instrumented?(mod) + mod.instance_variable_defined?(PROXY_IVAR) + end + + # Returns the proxy module (if any) of `mod`. + def self.proxy_module(mod) + mod.instance_variable_get(PROXY_IVAR) + end + # Instruments a method. # # type - The type (:class or :instance) of method to instrument. @@ -99,9 +115,8 @@ module Gitlab def self.instrument(type, mod, name) return unless Metrics.enabled? - name = name.to_sym - alias_name = :"_original_#{name}" - target = type == :instance ? mod : mod.singleton_class + name = name.to_sym + target = type == :instance ? mod : mod.singleton_class if type == :instance target = mod @@ -113,6 +128,12 @@ module Gitlab method = mod.method(name) end + unless instrumented?(target) + target.instance_variable_set(PROXY_IVAR, Module.new) + end + + proxy_module = self.proxy_module(target) + # Some code out there (e.g. the "state_machine" Gem) checks the arity of # a method to make sure it only passes arguments when the method expects # any. If we were to always overwrite a method to take an `*args` @@ -125,33 +146,17 @@ module Gitlab args_signature = '*args, &block' end - send_signature = "__send__(#{alias_name.inspect}, #{args_signature})" - - target.class_eval <<-EOF, __FILE__, __LINE__ + 1 - alias_method #{alias_name.inspect}, #{name.inspect} - + proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) - trans = Gitlab::Metrics::Instrumentation.transaction - - if trans - start = Time.now - retval = #{send_signature} - duration = (Time.now - start) * 1000.0 - - if duration >= Gitlab::Metrics.method_call_threshold - trans.increment(:method_duration, duration) - - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration }, - method: #{label.inspect}) - end - - retval + if trans = Gitlab::Metrics::Instrumentation.transaction + trans.measure_method(#{label.inspect}) { super } else - #{send_signature} + super end end EOF + + target.prepend(proxy_module) end # Small layer of indirection to make it easier to stub out the current diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb new file mode 100644 index 00000000000..faf0d9b6318 --- /dev/null +++ b/lib/gitlab/metrics/method_call.rb @@ -0,0 +1,52 @@ +module Gitlab + module Metrics + # Class for tracking timing information about method calls + class MethodCall + attr_reader :real_time, :cpu_time, :call_count + + # name - The full name of the method (including namespace) such as + # `User#sign_in`. + # + # series - The series to use for storing the data. + def initialize(name, series) + @name = name + @series = series + @real_time = 0.0 + @cpu_time = 0.0 + @call_count = 0 + end + + # Measures the real and CPU execution time of the supplied block. + def measure + start_real = Time.now + start_cpu = System.cpu_time + retval = yield + + @real_time += (Time.now - start_real) * 1000.0 + @cpu_time += System.cpu_time.to_f - start_cpu + @call_count += 1 + + retval + end + + # Returns a Metric instance of the current method call. + def to_metric + Metric.new( + @series, + { + duration: real_time, + cpu_duration: cpu_time, + call_count: call_count + }, + method: @name + ) + end + + # Returns true if the total runtime of this method exceeds the method call + # threshold. + def above_threshold? + real_time >= Metrics.method_call_threshold + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 7ea9555cc8c..1cd1ca30f70 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric + JITTER_RANGE = 0.000001..0.001 + attr_reader :series, :values, :tags, :created_at # series - The name of the series (as a String) to store the metric in. @@ -16,11 +18,29 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash + # InfluxDB overwrites an existing point if a new point has the same + # series, tag set, and timestamp. In a highly concurrent environment + # this means that using the number of seconds since the Unix epoch is + # inevitably going to collide with another timestamp. For example, two + # Rails requests processed by different processes may end up generating + # metrics using the _exact_ same timestamp (in seconds). + # + # Due to the way InfluxDB is set up there's no solution to this problem, + # all we can do is lower the amount of collisions. We do this by using + # Time#to_f which returns the seconds as a Float providing greater + # accuracy. We then add a small random value that is large enough to + # distinguish most timestamps but small enough to not alter the amount + # of seconds. + # + # See https://gitlab.com/gitlab-com/operations/issues/175 for more + # information. + time = @created_at.to_f + rand(JITTER_RANGE) + { series: @series, tags: @tags, values: @values, - timestamp: @created_at.to_i * 1_000_000_000 + timestamp: (time * 1_000_000_000).to_i } end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 6f179789d3e..e61670f491c 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -1,8 +1,9 @@ module Gitlab module Metrics - # Rack middleware for tracking Rails requests. + # Rack middleware for tracking Rails and Grape requests. class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' + ENDPOINT_KEY = 'api.endpoint' def initialize(app) @app = app @@ -21,6 +22,8 @@ module Gitlab ensure if env[CONTROLLER_KEY] tag_controller(trans, env) + elsif env[ENDPOINT_KEY] + tag_endpoint(trans, env) end trans.finish @@ -32,7 +35,7 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, filtered_path(env)) trans.set(:request_method, env['REQUEST_METHOD']) trans @@ -42,6 +45,30 @@ module Gitlab controller = env[CONTROLLER_KEY] trans.action = "#{controller.class.name}##{controller.action_name}" end + + def tag_endpoint(trans, env) + endpoint = env[ENDPOINT_KEY] + path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] + trans.action = "Grape##{endpoint.route.route_method} #{path}" + end + + private + + def filtered_path(env) + ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] + end + + def endpoint_paths_cache + @endpoint_paths_cache ||= Hash.new do |hash, http_method| + hash[http_method] = Hash.new do |inner_hash, raw_path| + inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) + end + end + end + + def endpoint_instrumentable_path(raw_path) + raw_path.sub('(.:format)', '').sub('/:version', '') + end end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index fc709222a9b..0000450d9bb 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -66,7 +66,11 @@ module Gitlab def sample_objects sample = Allocations.to_hash counts = sample.each_with_object({}) do |(klass, count), hash| - hash[klass.name] = count + name = klass.name + + next unless name + + hash[name] = count end # Symbols aren't allocated so we'll need to add those manually. diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 8008b3bc895..96cad941d5c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,7 @@ module Gitlab return unless current_transaction current_transaction.increment(:sql_duration, event.duration) + current_transaction.increment(:sql_count, 1) end private diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb new file mode 100644 index 00000000000..8e345e8ae4a --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -0,0 +1,41 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in Rails cache calls + class RailsCache < ActiveSupport::Subscriber + attach_to :active_support + + def cache_read(event) + increment(:cache_read, event.duration) + end + + def cache_write(event) + increment(:cache_write, event.duration) + end + + def cache_delete(event) + increment(:cache_delete, event.duration) + end + + def cache_exist?(event) + increment(:cache_exists, event.duration) + end + + def increment(key, duration) + return unless current_transaction + + current_transaction.increment(:cache_duration, duration) + current_transaction.increment(:cache_count, 1) + current_transaction.increment("#{key}_duration".to_sym, duration) + current_transaction.increment("#{key}_count".to_sym, 1) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 83371265278..a7d183b2f94 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -30,6 +30,17 @@ module Gitlab 0 end end + + # THREAD_CPUTIME is not supported on OS X + if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) + def self.cpu_time + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + end + else + def self.cpu_time + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + end + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 2578ddc49f4..4bc5081aa03 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values + attr_reader :tags, :values, :methods attr_accessor :action @@ -16,6 +16,7 @@ module Gitlab # plus method name. def initialize(action = nil) @metrics = [] + @methods = {} @started_at = nil @finished_at = nil @@ -51,9 +52,23 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + end + + # Measures the time it takes to execute a method. + # + # Multiple calls to the same method add up to the total runtime of the + # method. + # + # name - The full name of the method to measure (e.g. `User#sign_in`). + def measure_method(name, &block) + unless @methods[name] + series = "#{series_prefix}#{Instrumentation::SERIES}" + + @methods[name] = MethodCall.new(name, series) + end - @metrics << Metric.new("#{prefix}#{series}", values, tags) + @methods[name].measure(&block) end def increment(name, value) @@ -84,7 +99,13 @@ module Gitlab end def submit - metrics = @metrics.map do |metric| + submit = @metrics.dup + + @methods.each do |name, method| + submit << method.to_metric if method.above_threshold? + end + + submit_hashes = submit.map do |metric| hash = metric.to_hash hash[:tags][:action] ||= @action if @action @@ -92,12 +113,16 @@ module Gitlab hash end - Metrics.submit_metrics(metrics) + Metrics.submit_metrics(submit_hashes) end def sidekiq? Sidekiq.server? end + + def series_prefix + sidekiq? ? 'sidekiq_' : 'rails_' + end end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 50b0dd32380..5764ab15652 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -39,7 +39,7 @@ module Gitlab request_url = URI.join(base_url, project_path) domain_path = strip_url(request_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n"; + "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n" end def strip_url(url) diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb new file mode 100644 index 00000000000..56608b1b276 --- /dev/null +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to measure the latency between +# gitlab-workhorse forwarding a request to the Rails application and the +# time this middleware is reached. + +module Gitlab + module Middleware + class RailsQueueDuration + def initialize(app) + @app = app + end + + def call(env) + trans = Gitlab::Metrics.current_transaction + proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + if trans && proxy_start + # Time in milliseconds since gitlab-workhorse started the request + trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index 71cf6a0d886..8bdc89a7751 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -41,7 +41,7 @@ module Gitlab data[:issue] = note.noteable.hook_attrs elsif note.for_merge_request? data[:merge_request] = note.noteable.hook_attrs - elsif note.for_project_snippet? + elsif note.for_snippet? data[:snippet] = note.noteable.hook_attrs end @@ -59,8 +59,7 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } - base_data[:object_attributes][:url] = - Gitlab::UrlBuilder.new(:note).build(note.id) + base_data[:object_attributes][:url] = Gitlab::UrlBuilder.build(note) base_data end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 832fb08a526..78f3ecb4cb4 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -54,6 +54,12 @@ module Gitlab @user ||= build_new_user end + if external_provider? && @user + @user.external = true + elsif @user + @user.external = false + end + @user end @@ -63,13 +69,20 @@ module Gitlab return unless ldap_person # If a corresponding person exists with same uid in a LDAP server, - # set up a Gitlab user with dual LDAP and Omniauth identities. - if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) - # Case when a LDAP user already exists in Gitlab. Add the Omniauth identity to existing account. + # check if the user already has a GitLab account. + user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) + if user + # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. + log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) else - # No account in Gitlab yet: create it and add the LDAP identity - user = build_new_user + log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." + user = find_by_uid_and_provider + if user.nil? + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + user = build_new_user + end + log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) end @@ -113,6 +126,10 @@ module Gitlab end end + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0607a8b9592..183bd10d6a3 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,7 +2,8 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(project, query, repository_ref = nil) + def initialize(current_user, project, query, repository_ref = nil) + @current_user = current_user @project = project @repository_ref = if repository_ref.present? repository_ref @@ -73,7 +74,7 @@ module Gitlab end def notes - project.notes.user.search(query).order('updated_at DESC') + project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') end def commits diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index 97d1edab9c1..c8f12577112 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -36,11 +36,12 @@ module Gitlab commit.hook_attrs(with_changed_files: true) end - type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data data = { object_kind: type, + event_name: type, before: oldrev, after: newrev, ref: ref, @@ -65,7 +66,7 @@ module Gitlab # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, nil, 3) + commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" build(project, user, commits.last.id, commits.first.id, ref, commits) end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..40766f35f77 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,50 @@ +module Gitlab + class Redis + CACHE_NAMESPACE = 'cache:gitlab' + SESSION_NAMESPACE = 'session:gitlab' + SIDEKIQ_NAMESPACE = 'resque:gitlab' + + attr_reader :url + + # To be thread-safe we must be careful when writing the class instance + # variables @url and @pool. Because @pool depends on @url we need two + # mutexes to prevent deadlock. + URL_MUTEX = Mutex.new + POOL_MUTEX = Mutex.new + private_constant :URL_MUTEX, :POOL_MUTEX + + def self.url + @url || URL_MUTEX.synchronize { @url = new.url } + end + + def self.with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(url: url) } + end + end + @pool.with { |redis| yield redis } + end + + def self.redis_store_options + url = new.url + redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exist?(config_file) + @url = YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb deleted file mode 100644 index 4949c6db539..00000000000 --- a/lib/gitlab/redis_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - class RedisConfig - attr_reader :url - - def self.url - new.url - end - - def self.redis_store_options - url = new.url - redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) - if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path - end - redis_config_hash - end - - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) - - @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] - end - end - end -end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 4d830aa45e1..11c0b01f0dc 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,12 +1,12 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil, author = nil) + def initialize(project, current_user = nil) @project = project @current_user = current_user - @author = author @references = {} @@ -17,24 +17,37 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label milestone merge_request snippet commit commit_range).each do |type| + def references(type) + super(type, project, current_user) + end + + REFERABLES.each do |type| define_method("#{type}s") do - @references[type] ||= references(type, reference_context) + @references[type] ||= references(type) end end def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue, reference_context) + @references[:external_issue] ||= references(:external_issue) else - @references[:issue] ||= references(:issue, reference_context) + @references[:issue] ||= references(:issue) end end - private + def all + REFERABLES.each { |referable| send(referable.to_s.pluralize) } + @references.values.flatten + end + + def self.references_pattern + return @pattern if @pattern + + patterns = REFERABLES.map do |ref| + ref.to_s.classify.constantize.try(:reference_pattern) + end - def reference_context - { project: project, current_user: current_user, author: author } + @pattern = Regexp.union(patterns.compact) end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..c84c68f96f6 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,17 @@ module Gitlab (?<![\/.]) (?# rule #6-7) }x.freeze end + + def container_registry_reference_regex + git_reference_regex + end + + def environment_name_regex + @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + end + + def environment_name_regex_message + "can contain only letters, digits, '-' and '_'." + end end end diff --git a/lib/gitlab/repository_check_logger.rb b/lib/gitlab/repository_check_logger.rb new file mode 100644 index 00000000000..485b596ca57 --- /dev/null +++ b/lib/gitlab/repository_check_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class RepositoryCheckLogger < Gitlab::Logger + def self.file_name_noext + 'repocheck' + end + end +end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb new file mode 100644 index 00000000000..5132177de51 --- /dev/null +++ b/lib/gitlab/routing.rb @@ -0,0 +1,13 @@ +module Gitlab + module Routing + # Returns the URL helpers Module. + # + # This method caches the output as Rails' "url_helpers" method creates an + # anonymous module every time it's called. + # + # Returns a Module. + def self.url_helpers + @url_helpers ||= Gitlab::Application.routes.url_helpers + end + end +end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb new file mode 100644 index 00000000000..32c1c9ec5bb --- /dev/null +++ b/lib/gitlab/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Saml + class AuthHash < Gitlab::OAuth::AuthHash + + def groups + get_raw(Gitlab::Saml::Config.groups) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + + end + end +end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb new file mode 100644 index 00000000000..0f40c00f547 --- /dev/null +++ b/lib/gitlab/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Saml + class Config + + class << self + def options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + + end + end +end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index b1e30110ef5..8943022612c 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -12,13 +12,13 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider - if auto_link_ldap_user? @user ||= find_or_create_ldap_user end - if auto_link_saml_enabled? + @user ||= find_by_uid_and_provider + + if auto_link_saml_user? @user ||= find_by_email end @@ -26,6 +26,16 @@ module Gitlab @user ||= build_new_user end + if external_users_enabled? && @user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + @user.external = false + else + @user.external = true + end + end + @user end @@ -37,11 +47,24 @@ module Gitlab end end + def changed? + return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + protected - def auto_link_saml_enabled? + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end + + def external_users_enabled? + !Gitlab::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) + end end end end diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb new file mode 100644 index 00000000000..8304b9a482c --- /dev/null +++ b/lib/gitlab/sanitizers/svg.rb @@ -0,0 +1,57 @@ +module Gitlab + module Sanitizers + module SVG + def self.clean(data) + Loofah.xml_document(data).scrub!(Scrubber.new).to_s + end + + class Scrubber < Loofah::Scrubber + # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes + DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u + + def scrub(node) + unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) + node.unlink + return + end + + valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name] + return unless valid_attributes + + node.attribute_nodes.each do |attr| + attr_name = attribute_name_with_namespace(attr) + + if valid_attributes.include?(attr_name) + attr.unlink if unsafe_href?(attr) + else + # Arbitrary data attributes are allowed. + unless allows_data_attribute?(node) && data_attribute?(attr) + attr.unlink + end + end + end + end + + def attribute_name_with_namespace(attr) + if attr.namespace + "#{attr.namespace.prefix}:#{attr.name}" + else + attr.name + end + end + + def allows_data_attribute?(node) + Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name) + end + + def unsafe_href?(attr) + attribute_name_with_namespace(attr) == 'xlink:href' && !attr.value.start_with?('#') + end + + def data_attribute?(attr) + attr.name.start_with?('data-') && attr.name =~ DATA_ATTR_PATTERN && attr.namespace.nil? + end + end + end + end +end diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb new file mode 100644 index 00000000000..7b6b70d8dbc --- /dev/null +++ b/lib/gitlab/sanitizers/svg/whitelist.rb @@ -0,0 +1,109 @@ +# Generated from: +# SVG element list: https://www.w3.org/TR/SVG/eltindex.html +# SVG Attribute list: https://www.w3.org/TR/SVG/attindex.html +module Gitlab + module Sanitizers + module SVG + class Whitelist + ALLOWED_ELEMENTS = %w[ + a altGlyph altGlyphDef altGlyphItem animate + animateColor animateMotion animateTransform circle clipPath color-profile + cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer + feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap + feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur + feImage feMerge feMergeNode feMorphology feOffset fePointLight + feSpecularLighting feSpotLight feTile feTurbulence filter font font-face + font-face-format font-face-name font-face-src font-face-uri foreignObject + g glyph glyphRef hkern image line linearGradient marker mask metadata + missing-glyph mpath path pattern polygon polyline radialGradient rect + script set stop style svg switch symbol text textPath title tref tspan use + view vkern].freeze + + ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze + + ALLOWED_ATTRIBUTES = { + 'a' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'altGlyphDef' => %w[id xml:base xml:lang xml:space], + 'altGlyphItem' => %w[id xml:base xml:lang xml:space], + 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'desc' => %w[class id style xml:base xml:lang xml:space], + 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], + 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], + 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feMergeNode' => %w[id xml:base xml:lang xml:space], + 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], + 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], + 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], + 'font-face-format' => %w[id string xml:base xml:lang xml:space], + 'font-face-name' => %w[id name xml:base xml:lang xml:space], + 'font-face-src' => %w[id xml:base xml:lang xml:space], + 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], + 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], + 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], + 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'metadata' => %w[id xml:base xml:lang xml:space], + 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'style' => %w[id media title type xml:base xml:lang xml:space], + 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], + 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'title' => %w[class id style xml:base xml:lang xml:space], + 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], + 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], + 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] + }.freeze + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index f13528a2eea..f8ab2b1f09e 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,12 +1,13 @@ module Gitlab class SearchResults - attr_reader :query + attr_reader :current_user, :query # Limit search results by passed projects # It allows us to search only for projects user has access to attr_reader :limit_projects - def initialize(limit_projects, query) + def initialize(current_user, limit_projects, query) + @current_user = current_user @limit_projects = limit_projects || Project.all @query = Shellwords.shellescape(query) if query.present? end @@ -58,7 +59,7 @@ module Gitlab end def issues - issues = Issue.where(project_id: project_ids_relation) + issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) if query =~ /#(\d+)\z/ issues = issues.where(iid: $1) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 2ef0e982256..7cf506ebe64 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -5,7 +5,7 @@ module Gitlab SeedFu.quiet = true yield SeedFu.quiet = false - puts "\nOK".green + puts "\nOK".color(:green) end def self.by_user(user) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 37232743325..ae85b294d31 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -29,8 +29,8 @@ module Gitlab "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) - Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" - Process.kill('SIGUSR1', Process.pid) + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Process.kill('SIGTERM', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 6f0d02cafd1..fe65c246101 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,56 +1,68 @@ module Gitlab class UrlBuilder - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers include GitlabRoutingHelper + include ActionView::RecordIdentifier - def initialize(type) - @type = type - end + attr_reader :object - def build(id) - case @type - when :issue - build_issue_url(id) - when :merge_request - build_merge_request_url(id) - when :note - build_note_url(id) + def self.build(object) + new(object).url + end + def url + case object + when Commit + commit_url + when Issue + issue_url(object) + when MergeRequest + merge_request_url(object) + when Note + note_url + when WikiPage + wiki_page_url + else + raise NotImplementedError.new("No URL builder defined for #{object.class}") end end private - def build_issue_url(id) - issue = Issue.find(id) - issue_url(issue) + def initialize(object) + @object = object end - def build_merge_request_url(id) - merge_request = MergeRequest.find(id) - merge_request_url(merge_request) + def commit_url(opts = {}) + return '' if object.project.nil? + + namespace_project_commit_url({ + namespace_id: object.project.namespace, + project_id: object.project, + id: object.id + }.merge!(opts)) end - def build_note_url(id) - note = Note.find(id) - if note.for_commit? - namespace_project_commit_url(namespace_id: note.project.namespace, - id: note.commit_id, - project_id: note.project, - anchor: "note_#{note.id}") - elsif note.for_issue? - issue = Issue.find(note.noteable_id) - issue_url(issue, - anchor: "note_#{note.id}") - elsif note.for_merge_request? - merge_request = MergeRequest.find(note.noteable_id) - merge_request_url(merge_request, - anchor: "note_#{note.id}") - elsif note.for_project_snippet? - snippet = Snippet.find(note.noteable_id) - project_snippet_url(snippet, - anchor: "note_#{note.id}") + def note_url + if object.for_commit? + commit_url(id: object.commit_id, anchor: dom_id(object)) + + elsif object.for_issue? + issue = Issue.find(object.noteable_id) + issue_url(issue, anchor: dom_id(object)) + + elsif object.for_merge_request? + merge_request = MergeRequest.find(object.noteable_id) + merge_request_url(merge_request, anchor: dom_id(object)) + + elsif object.for_snippet? + snippet = Snippet.find(object.noteable_id) + project_snippet_url(snippet, anchor: dom_id(object)) end end + + def wiki_page_url + namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb new file mode 100644 index 00000000000..7d02fe3c971 --- /dev/null +++ b/lib/gitlab/url_sanitizer.rb @@ -0,0 +1,54 @@ +module Gitlab + class UrlSanitizer + def self.sanitize(content) + regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) + + content.gsub(regexp) { |url| new(url).masked_url } + end + + def initialize(url, credentials: nil) + @url = Addressable::URI.parse(url) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def masked_url + url = @url.dup + url.password = "*****" unless url.password.nil? + url.user = "*****" unless url.user.nil? + url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 3160a3c7582..9462f3368e6 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -6,6 +6,14 @@ module Gitlab module VisibilityLevel extend CurrentSettings + extend ActiveSupport::Concern + + included do + scope :public_only, -> { where(visibility_level: PUBLIC) } + scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + + scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } + end PRIVATE = 0 unless const_defined?(:PRIVATE) INTERNAL = 10 unless const_defined?(:INTERNAL) @@ -24,6 +32,13 @@ module Gitlab } end + def highest_allowed_level + restricted_levels = current_application_settings.restricted_visibility_levels + + allowed_levels = self.values - restricted_levels + allowed_levels.max || PRIVATE + end + def allowed_for?(user, level) user.is_admin? || allowed_level?(level.to_i) end @@ -48,10 +63,6 @@ module Gitlab options.has_value?(level) end - def allowed_fork_levels(origin_level) - [PRIVATE, INTERNAL, PUBLIC].select{ |level| level <= origin_level } - end - def level_name(level) level_name = 'Unknown' options.each do |name, lvl| diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c3ddd4c2680..40e8299c36b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -6,6 +6,13 @@ module Gitlab SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' class << self + def git_http_ok(repository, user) + { + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'RepoPath' => repository.path_to_repo, + } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -14,24 +21,39 @@ module Gitlab [ SEND_DATA_HEADER, - "git-blob:#{encode(params)}", + "git-blob:#{encode(params)}" ] end - def send_git_archive(project, ref, format) + def send_git_archive(repository, ref:, format:) format ||= 'tar.gz' format.downcase! - params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? [ SEND_DATA_HEADER, - "git-archive:#{encode(params)}", + "git-archive:#{encode(params)}" ] end - + + def send_git_diff(repository, diff_refs) + from, to = diff_refs + + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => from.sha, + 'ShaTo' => to.sha + } + + [ + SEND_DATA_HEADER, + "git-diff:#{encode(params)}" + ] + end + protected - + def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb new file mode 100644 index 00000000000..d6d6af7089c --- /dev/null +++ b/lib/json_web_token/rsa_token.rb @@ -0,0 +1,42 @@ +module JSONWebToken + class RSAToken < Token + attr_reader :key_file + + def initialize(key_file) + super() + @key_file = key_file + end + + def encoded + headers = { + kid: kid + } + JWT.encode(payload, key, 'RS256', headers) + end + + private + + def key_data + @key_data ||= File.read(key_file) + end + + def key + @key ||= OpenSSL::PKey::RSA.new(key_data) + end + + def public_key + key.public_key + end + + def kid + # calculate sha256 from DER encoded ASN1 + kid = Digest::SHA256.digest(public_key.to_der) + + # we encode only 30 bytes with base32 + kid = Base32.encode(kid[0..29]) + + # insert colon every 4 characters + kid.scan(/.{4}/).join(':') + end + end +end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb new file mode 100644 index 00000000000..5b67715b0b2 --- /dev/null +++ b/lib/json_web_token/token.rb @@ -0,0 +1,46 @@ +module JSONWebToken + class Token + attr_accessor :issuer, :subject, :audience, :id + attr_accessor :issued_at, :not_before, :expire_time + + def initialize + @id = SecureRandom.uuid + @issued_at = Time.now + # we give a few seconds for time shift + @not_before = issued_at - 5.seconds + # default 60 seconds should be more than enough for this authentication token + @expire_time = issued_at + 1.minute + @custom_payload = {} + end + + def [](key) + @custom_payload[key] + end + + def []=(key, value) + @custom_payload[key] = value + end + + def encoded + raise NotImplementedError + end + + def payload + @custom_payload.merge(default_payload) + end + + private + + def default_payload + { + jti: id, + aud: audience, + sub: subject, + iss: issuer, + iat: issued_at.to_i, + nbf: not_before.to_i, + exp: expire_time.to_i + }.compact + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index d95e7023d2e..31b00ff128a 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -173,7 +173,7 @@ check_stale_pids(){ fi fi if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then - echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + echo "Removing stale GitLab Workhorse pid. This is most likely caused by GitLab Workhorse crashing the last time it ran." if ! rm "$gitlab_workhorse_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 @@ -208,7 +208,7 @@ start_gitlab() { echo "Starting GitLab Sidekiq" fi if [ "$gitlab_workhorse_status" != "0" ]; then - echo "Starting gitlab-workhorse" + echo "Starting GitLab Workhorse" fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" @@ -232,7 +232,7 @@ start_gitlab() { fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The gitlab-workhorse is already running with pid $spid, not restarting" + echo "The GitLab Workhorse is already running with pid $spid, not restarting" else # No need to remove a socket, gitlab-workhorse does this itself. # Because gitlab-workhorse has multiple executables we need to fix @@ -271,7 +271,7 @@ stop_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "Shutting down gitlab-workhorse" + echo "Shutting down GitLab Workhorse" kill -- $(cat $gitlab_workhorse_pid_path) fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then @@ -320,9 +320,9 @@ print_status() { printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The gitlab-workhorse with pid $hpid is running." + echo "The GitLab Workhorse with pid $hpid is running." else - printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" + printf "The GitLab Workhorse is \033[31mnot running\033[0m.\n" fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 1324e4cd267..d521de28e8a 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -61,7 +61,8 @@ server { error_page 422 /422.html; error_page 500 /500.html; error_page 502 /502.html; - location ~ ^/(404|422|500|502)\.html$ { + error_page 503 /503.html; + location ~ ^/(404|422|500|502|503)\.html$ { root /home/git/gitlab/public; internal; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index af6ea9ed706..bf014b56cf6 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -105,7 +105,8 @@ server { error_page 422 /422.html; error_page 500 /500.html; error_page 502 /502.html; - location ~ ^/(404|422|500|502)\.html$ { + error_page 503 /503.html; + location ~ ^/(404|422|500|502|503)\.html$ { root /home/git/gitlab/public; internal; } diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci deleted file mode 100644 index bf05edfd780..00000000000 --- a/lib/support/nginx/gitlab_ci +++ /dev/null @@ -1,29 +0,0 @@ -# GITLAB CI -server { - listen 80 default_server; # e.g., listen 192.168.1.1:80; - server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com; - - access_log /var/log/nginx/gitlab_ci_access.log; - error_log /var/log/nginx/gitlab_ci_error.log; - - # expose API to fix runners - location /api { - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - - # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN - resolver 8.8.8.8 8.8.4.4; - proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # redirect all other CI requests - location / { - return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # adjust this to match the largest build log your runners might submit, - # set to 0 to disable limit - client_max_body_size 10m; -}
\ No newline at end of file diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl new file mode 100644 index 00000000000..92511e26861 --- /dev/null +++ b/lib/support/nginx/registry-ssl @@ -0,0 +1,53 @@ +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### + +## Redirects all HTTP traffic to the HTTPS host +server { + listen *:80; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + return 301 https://$http_host:$request_uri; + access_log /var/log/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/nginx/gitlab_registry_error.log; +} + +server { + # If a different port is specified in https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/config/gitlab.yml.example#L182, + # it should be declared here as well + listen *:443 ssl http2; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + + client_max_body_size 0; + chunked_transfer_encoding on; + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt + ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key + + ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4'; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache builtin:1000 shared:SSL:10m; + ssl_session_timeout 5m; + + access_log /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/gitlab/nginx/gitlab_registry_error.log; + + location / { + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass http://localhost:5000; + } + +} diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 51e746ef923..2214f855200 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -4,18 +4,19 @@ namespace :cache do desc "GitLab | Clear redis cache" task :clear => :environment do - redis = Redis.new(url: Gitlab::RedisConfig.url) - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index cfaf4a129b1..030ee8bafcb 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,19 +1,50 @@ -# This task will generate a standard and Retina sprite of all of the current -# Gemojione Emojis, with the accompanying SCSS map. -# -# It will not appear in `rake -T` output, and the dependent gems are not -# included in the Gemfile by default, because this task will only be needed -# occasionally, such as when new Emojis are added to Gemojione. - -begin - require 'sprite_factory' - require 'rmagick' -rescue LoadError - # noop -end - namespace :gemojione do + desc 'Generates Emoji SHA256 digests' + task digests: :environment do + require 'digest/sha2' + require 'json' + + dir = Gemojione.index.images_path + digests = [] + aliases = Hash.new { |hash, key| hash[key] = [] } + aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + + JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| + aliases[real_name] << alias_name + end + + AwardEmoji.emojis.map do |name, emoji_hash| + fpath = File.join(dir, "#{emoji_hash['unicode']}.png") + digest = Digest::SHA256.file(fpath).hexdigest + + digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + + aliases[name].each do |alias_name| + digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + end + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(digests)) + end + end + + # This task will generate a standard and Retina sprite of all of the current + # Gemojione Emojis, with the accompanying SCSS map. + # + # It will not appear in `rake -T` output, and the dependent gems are not + # included in the Gemfile by default, because this task will only be needed + # occasionally, such as when new Emojis are added to Gemojione. task sprite: :environment do + begin + require 'sprite_factory' + require 'rmagick' + rescue LoadError + # noop + end + check_requirements! SIZE = 20 diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index cb4abe13799..9ee72fde92f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,6 +14,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke + Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -22,7 +23,7 @@ namespace :gitlab do end # Restore backup of GitLab system - desc "GitLab | Restore a previously created backup" + desc 'GitLab | Restore a previously created backup' task restore: :environment do warn_user_is_not_gitlab configure_cron_mode @@ -30,128 +31,174 @@ namespace :gitlab do backup = Backup::Manager.new backup.unpack - Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") - Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") - Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") - Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") - Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") - Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") - Rake::Task["gitlab:shell:setup"].invoke + unless backup.skipped?('db') + unless ENV['force'] == 'yes' + warning = warning = <<-MSG.strip_heredoc + Before restoring the database we recommend removing all existing + tables to avoid future upgrade problems. Be aware that if you have + custom tables in the GitLab database these tables and all data will be + removed. + MSG + ask_to_continue + puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + $progress.puts 'Cleaning the database ... '.color(:blue) + Rake::Task['gitlab:db:drop_tables'].invoke + $progress.puts 'done'.color(:green) + Rake::Task['gitlab:backup:db:restore'].invoke + end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') + Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') + Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') + Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') + Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') + Rake::Task['gitlab:shell:setup'].invoke backup.cleanup end namespace :repo do task create: :environment do - $progress.puts "Dumping repositories ...".blue + $progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Repository.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring repositories ...".blue + $progress.puts "Restoring repositories ...".color(:blue) Backup::Repository.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :db do task create: :environment do - $progress.puts "Dumping database ... ".blue + $progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Database.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring database ... ".blue + $progress.puts "Restoring database ... ".color(:blue) Backup::Database.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :builds do task create: :environment do - $progress.puts "Dumping builds ... ".blue + $progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Builds.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring builds ... ".blue + $progress.puts "Restoring builds ... ".color(:blue) Backup::Builds.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :uploads do task create: :environment do - $progress.puts "Dumping uploads ... ".blue + $progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Uploads.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring uploads ... ".blue + $progress.puts "Restoring uploads ... ".color(:blue) Backup::Uploads.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :artifacts do task create: :environment do - $progress.puts "Dumping artifacts ... ".blue + $progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Artifacts.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring artifacts ... ".blue + $progress.puts "Restoring artifacts ... ".color(:blue) Backup::Artifacts.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :lfs do task create: :environment do - $progress.puts "Dumping lfs objects ... ".blue + $progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Lfs.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring lfs objects ... ".blue + $progress.puts "Restoring lfs objects ... ".color(:blue) Backup::Lfs.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) + end + end + + namespace :registry do + task create: :environment do + $progress.puts "Dumping container registry images ... ".color(:blue) + + if Gitlab.config.registry.enabled + if ENV["SKIP"] && ENV["SKIP"].include?("registry") + $progress.puts "[SKIPPED]".color(:cyan) + else + Backup::Registry.new.dump + $progress.puts "done".color(:green) + end + else + $progress.puts "[DISABLED]".color(:cyan) + end + end + + task restore: :environment do + $progress.puts "Restoring container registry images ... ".color(:blue) + if Gitlab.config.registry.enabled + Backup::Registry.new.restore + $progress.puts "done".color(:green) + else + $progress.puts "[DISABLED]".color(:cyan) + end end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 27ed57efe55..12d6ac45fb6 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -50,14 +50,14 @@ namespace :gitlab do end if correct_options.all? - puts "yes".green + puts "yes".color(:green) else print "Trying to fix Git error automatically. ..." if auto_fix_git_config(options) - puts "Success".green + puts "Success".color(:green) else - puts "Failed".red + puts "Failed".color(:red) try_fixing_it( sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") ) @@ -74,9 +74,9 @@ namespace :gitlab do database_config_file = Rails.root.join("config", "database.yml") if File.exists?(database_config_file) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Copy config/database.yml.<your db> to config/database.yml", "Check that the information in config/database.yml is correct" @@ -95,9 +95,9 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") if File.exists?(gitlab_config_file) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Copy config/gitlab.yml.example to config/gitlab.yml", "Update config/gitlab.yml to match your setup" @@ -114,14 +114,14 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") unless File.exists?(gitlab_config_file) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) end # omniauth or ldap could have been deleted from the file unless Gitlab.config['git_host'] - puts "no".green + puts "no".color(:green) else - puts "yes".red + puts "yes".color(:red) try_fixing_it( "Backup your config/gitlab.yml", "Copy config/gitlab.yml.example to config/gitlab.yml", @@ -138,16 +138,16 @@ namespace :gitlab do print "Init script exists? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end script_path = "/etc/init.d/gitlab" if File.exists?(script_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Install the init script" ) @@ -162,7 +162,7 @@ namespace :gitlab do print "Init script up-to-date? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end @@ -170,7 +170,7 @@ namespace :gitlab do script_path = "/etc/init.d/gitlab" unless File.exists?(script_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end @@ -178,9 +178,9 @@ namespace :gitlab do script_content = File.read(script_path) if recipe_content == script_content - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Redownload the init script" ) @@ -197,9 +197,9 @@ namespace :gitlab do migration_status, _ = Gitlab::Popen.popen(%W(bundle exec rake db:migrate:status)) unless migration_status =~ /down\s+\d{14}/ - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") ) @@ -210,13 +210,13 @@ namespace :gitlab do def check_orphaned_group_members print "Database contains orphaned GroupMembers? ... " if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".red + puts "yes".color(:red) try_fixing_it( "You can delete the orphaned records using something along the lines of:", sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") ) else - puts "no".green + puts "no".color(:green) end end @@ -226,9 +226,9 @@ namespace :gitlab do log_path = Rails.root.join("log") if File.writable?(log_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R gitlab #{log_path}", "sudo chmod -R u+rwX #{log_path}" @@ -246,9 +246,9 @@ namespace :gitlab do tmp_path = Rails.root.join("tmp") if File.writable?(tmp_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R gitlab #{tmp_path}", "sudo chmod -R u+rwX #{tmp_path}" @@ -264,7 +264,7 @@ namespace :gitlab do print "Uploads directory setup correctly? ... " unless File.directory?(Rails.root.join('public/uploads')) - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" ) @@ -280,16 +280,16 @@ namespace :gitlab do if File.stat(upload_path).mode == 040700 unless Dir.exists?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.magenta + puts 'skipped (no tmp uploads folder yet)'.color(:magenta) return end # If tmp upload dir has incorrect permissions, assume others do as well # Verify drwx------ permissions if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R #{gitlab_user} #{upload_path}", "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", @@ -301,9 +301,9 @@ namespace :gitlab do fix_and_rerun end else - puts "no".red + puts "no".color(:red) try_fixing_it( - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" + "sudo chmod 700 #{upload_path}" ) for_more_information( see_installation_guide_section "GitLab" @@ -320,9 +320,9 @@ namespace :gitlab do redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your redis server to a version >= #{min_redis_version}" ) @@ -361,10 +361,10 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path if File.exists?(repo_base_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red - puts "#{repo_base_path} is missing".red + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) try_fixing_it( "This should have been created when setting up GitLab Shell.", "Make sure it's set correctly in config/gitlab.yml", @@ -382,14 +382,14 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end unless File.symlink?(repo_base_path) - puts "no".green + puts "no".color(:green) else - puts "yes".red + puts "yes".color(:red) try_fixing_it( "Make sure it's set to the real directory in config/gitlab.yml" ) @@ -402,14 +402,14 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", @@ -429,17 +429,17 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end uid = uid_for(gitlab_shell_ssh_user) gid = gid_for(gitlab_shell_owner_group) if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".green + puts "yes".color(:green) else - puts "no".red - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".blue + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) try_fixing_it( "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" ) @@ -456,7 +456,7 @@ namespace :gitlab do gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path unless Project.count > 0 - puts "can't check, you have no projects".magenta + puts "can't check, you have no projects".color(:magenta) return end puts "" @@ -466,12 +466,12 @@ namespace :gitlab do project_hook_directory = File.join(project.repository.path_to_repo, "hooks") if project.empty_repo? - puts "repository is empty".magenta + puts "repository is empty".color(:magenta) elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) && (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path)) - puts 'ok'.green + puts 'ok'.color(:green) else - puts "wrong or missing hooks".red + puts "wrong or missing hooks".color(:red) try_fixing_it( sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), 'Check the hooks_path in config/gitlab.yml', @@ -491,9 +491,9 @@ namespace :gitlab do check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) puts "Running #{check_cmd}" if system(check_cmd, chdir: gitlab_shell_repo_base) - puts 'gitlab-shell self-check successful'.green + puts 'gitlab-shell self-check successful'.color(:green) else - puts 'gitlab-shell self-check failed'.red + puts 'gitlab-shell self-check failed'.color(:red) try_fixing_it( 'Make sure GitLab is running;', 'Check the gitlab-shell configuration file:', @@ -507,7 +507,7 @@ namespace :gitlab do print "projects have namespace: ... " unless Project.count > 0 - puts "can't check, you have no projects".magenta + puts "can't check, you have no projects".color(:magenta) return end puts "" @@ -516,9 +516,9 @@ namespace :gitlab do print sanitized_message(project) if project.namespace - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Migrate global projects" ) @@ -576,9 +576,9 @@ namespace :gitlab do print "Running? ... " if sidekiq_process_count > 0 - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("RAILS_ENV=production bin/background_jobs start") ) @@ -596,9 +596,9 @@ namespace :gitlab do print 'Number of Sidekiq processes ... ' if process_count == 1 - puts '1'.green + puts '1'.color(:green) else - puts "#{process_count}".red + puts "#{process_count}".color(:red) try_fixing_it( 'sudo service gitlab stop', "sudo pkill -u #{gitlab_user} -f sidekiq", @@ -623,7 +623,6 @@ namespace :gitlab do start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_address_formatted_correctly check_imap_authentication if Rails.env.production? @@ -643,34 +642,20 @@ namespace :gitlab do # Checks ######################## - def check_address_formatted_correctly - print "Address formatted correctly? ... " - - if Gitlab::IncomingEmail.address_formatted_correctly? - puts "yes".green - else - puts "no".red - try_fixing_it( - "Make sure that the address in config/gitlab.yml includes the '%{key}' placeholder." - ) - fix_and_rerun - end - end - def check_initd_configured_correctly print "Init.d configured correctly? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end path = "/etc/default/gitlab" if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Enable mail_room in the init.d configuration." ) @@ -687,9 +672,9 @@ namespace :gitlab do path = Rails.root.join("Procfile") if File.exist?(path) && File.read(path) =~ /^mail_room:/ - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Enable mail_room in your Procfile." ) @@ -706,14 +691,14 @@ namespace :gitlab do path = "/etc/default/gitlab" unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end if mail_room_running? - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("RAILS_ENV=production bin/mail_room start") ) @@ -744,9 +729,9 @@ namespace :gitlab do end if connected - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Check that the information in config/gitlab.yml is correct" ) @@ -814,7 +799,7 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".blue) + username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| @@ -826,7 +811,7 @@ namespace :gitlab do repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } else - puts "\nUser '#{username}' not found".red + puts "\nUser '#{username}' not found".color(:red) end end end @@ -835,13 +820,13 @@ namespace :gitlab do ########################## def fix_and_rerun - puts " Please #{"fix the error above"} and rerun the checks.".red + puts " Please #{"fix the error above"} and rerun the checks.".color(:red) end def for_more_information(*sources) sources = sources.shift if sources.first.is_a?(Array) - puts " For more information see:".blue + puts " For more information see:".color(:blue) sources.each do |source| puts " #{source}" end @@ -849,7 +834,7 @@ namespace :gitlab do def finished_checking(component) puts "" - puts "Checking #{component.yellow} ... #{"Finished".green}" + puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" puts "" end @@ -870,14 +855,14 @@ namespace :gitlab do end def start_checking(component) - puts "Checking #{component.yellow} ..." + puts "Checking #{component.color(:yellow)} ..." puts "" end def try_fixing_it(*steps) steps = steps.shift if steps.first.is_a?(Array) - puts " Try fixing it:".blue + puts " Try fixing it:".color(:blue) steps.each do |step| puts " #{step}" end @@ -889,9 +874,9 @@ namespace :gitlab do print "GitLab Shell version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "OK (#{current_version})".green + puts "OK (#{current_version})".color(:green) else - puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".red + puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end @@ -902,9 +887,9 @@ namespace :gitlab do print "Ruby version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".green + puts "yes (#{current_version})".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your ruby to a version >= #{required_version} from #{current_version}" ) @@ -920,9 +905,9 @@ namespace :gitlab do print "Git version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".green + puts "yes (#{current_version})".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your git to a version >= #{required_version} from #{current_version}" ) @@ -940,9 +925,9 @@ namespace :gitlab do def sanitized_message(project) if should_sanitize? - "#{project.namespace_id.to_s.yellow}/#{project.id.to_s.yellow} ... " + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " else - "#{project.name_with_namespace.yellow} ... " + "#{project.name_with_namespace.color(:yellow)} ... " end end @@ -955,7 +940,7 @@ namespace :gitlab do end def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.yellow}" + puts "\nChecking repo at #{repo_dir.color(:yellow)}" git_fsck(repo_dir) check_config_lock(repo_dir) @@ -963,25 +948,25 @@ namespace :gitlab do end def git_fsck(repo_dir) - puts "Running `git fsck`".yellow + puts "Running `git fsck`".color(:yellow) system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) end def check_config_lock(repo_dir) config_exists = File.exist?(File.join(repo_dir,'config.lock')) - config_output = config_exists ? 'yes'.red : 'no'.green - puts "'config.lock' file exists?".yellow + " ... #{config_output}" + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" end def check_ref_locks(repo_dir) lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) if lock_files.present? - puts "Ref lock files exist:".red + puts "Ref lock files exist:".color(:red) lock_files.each do |lock_file| puts " #{lock_file}" end else - puts "No ref lock files exist".green + puts "No ref lock files exist".color(:green) end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 9f5852ac613..ab0028d6603 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -10,7 +10,7 @@ namespace :gitlab do git_base_path = Gitlab.config.gitlab_shell.repos_path all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.yellow + puts git_base_path.color(:yellow) puts "Looking for directories to remove... " all_dirs.reject! do |dir| @@ -29,17 +29,17 @@ namespace :gitlab do if remove_flag if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".red + puts "Removed...#{dir_path}".color(:red) else - puts "Cannot remove #{dir_path}".red + puts "Cannot remove #{dir_path}".color(:red) end else - puts "Can be removed: #{dir_path}".red + puts "Can be removed: #{dir_path}".color(:red) end end unless remove_flag - puts "To cleanup this directories run this command with REMOVE=true".yellow + puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) end end @@ -75,19 +75,19 @@ namespace :gitlab do next unless user.ldap_user? print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." if Gitlab::LDAP::Access.allowed?(user) - puts " [OK]".green + puts " [OK]".color(:green) else if block_flag user.block! unless user.blocked? - puts " [BLOCKED]".red + puts " [BLOCKED]".color(:red) else - puts " [NOT IN LDAP]".yellow + puts " [NOT IN LDAP]".color(:yellow) end end end unless block_flag - puts "To block these users run this command with BLOCK=true".yellow + puts "To block these users run this command with BLOCK=true".color(:yellow) end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake new file mode 100644 index 00000000000..7230b9485be --- /dev/null +++ b/lib/tasks/gitlab/db.rake @@ -0,0 +1,50 @@ +namespace :gitlab do + namespace :db do + desc 'GitLab | Manually insert schema migration version' + task :mark_migration_complete, [:version] => :environment do |_, args| + unless args[:version] + puts "Must specify a migration version as an argument".color(:red) + exit 1 + end + + version = args[:version].to_i + if version == 0 + puts "Version '#{args[:version]}' must be a non-zero integer".color(:red) + exit 1 + end + + sql = "INSERT INTO schema_migrations (version) VALUES (#{version})" + begin + ActiveRecord::Base.connection.execute(sql) + puts "Successfully marked '#{version}' as complete".color(:green) + rescue ActiveRecord::RecordNotUnique + puts "Migration version '#{version}' is already marked complete".color(:yellow) + end + end + + desc 'Drop all tables' + task :drop_tables => :environment do + connection = ActiveRecord::Base.connection + tables = connection.tables + tables.delete 'schema_migrations' + # Truncate schema_migrations to ensure migrations re-run + connection.execute('TRUNCATE schema_migrations') + + # Drop tables with cascade to avoid dependent table errors + # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html + # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html + # Add `IF EXISTS` because cascade could have already deleted a table. + tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + end + + desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' + task configure: :environment do + if ActiveRecord::Base.connection.tables.any? + Rake::Task['db:migrate'].invoke + else + Rake::Task['db:schema:load'].invoke + Rake::Task['db:seed_fu'].invoke + end + end + end +end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 65ee430d550..f9834a4dae8 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -5,7 +5,7 @@ namespace :gitlab do task repack: :environment do failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -15,7 +15,7 @@ namespace :gitlab do task gc: :environment do failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -25,7 +25,7 @@ namespace :gitlab do task prune: :environment do failures = perform_git_cmd(%W(git prune), "Git Prune") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -47,7 +47,7 @@ namespace :gitlab do end def output_failures(failures) - puts "The following repositories reported errors:".red + puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 1c04f47f08f..4753f00c26a 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -23,7 +23,7 @@ namespace :gitlab do group_name, name = File.split(path) group_name = nil if group_name == '.' - puts "Processing #{repo_path}".yellow + puts "Processing #{repo_path}".color(:yellow) if path.end_with?('.wiki') puts " * Skipping wiki repo" @@ -51,9 +51,9 @@ namespace :gitlab do group.path = group_name group.owner = user if group.save - puts " * Created Group #{group.name} (#{group.id})".green + puts " * Created Group #{group.name} (#{group.id})".color(:green) else - puts " * Failed trying to create group #{group.name}".red + puts " * Failed trying to create group #{group.name}".color(:red) end end # set project group @@ -63,17 +63,17 @@ namespace :gitlab do project = Projects::CreateService.new(user, project_params).execute if project.persisted? - puts " * Created #{project.name} (#{repo_path})".green + puts " * Created #{project.name} (#{repo_path})".color(:green) project.update_repository_size project.update_commit_count else - puts " * Failed trying to create #{project.name} (#{repo_path})".red - puts " Errors: #{project.errors.messages}".red + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) end end end - puts "Done!".green + puts "Done!".color(:green) end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index d6883a563ee..352b566df24 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -15,15 +15,15 @@ namespace :gitlab do rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) puts "" - puts "System information".yellow - puts "System:\t\t#{os_name || "unknown".red}" + puts "System information".color(:yellow) + puts "System:\t\t#{os_name || "unknown".color(:red)}" puts "Current User:\t#{run(%W(whoami))}" - puts "Using RVM:\t#{rvm_version.present? ? "yes".green : "no"}" + puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? - puts "Ruby Version:\t#{ruby_version || "unknown".red}" - puts "Gem Version:\t#{gem_version || "unknown".red}" - puts "Bundler Version:#{bunder_version || "unknown".red}" - puts "Rake Version:\t#{rake_version || "unknown".red}" + puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" + puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" + puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" + puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" @@ -39,7 +39,7 @@ namespace :gitlab do omniauth_providers.map! { |provider| provider['name'] } puts "" - puts "GitLab information".yellow + puts "GitLab information".color(:yellow) puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab::REVISION}" puts "Directory:\t#{Rails.root}" @@ -47,9 +47,9 @@ namespace :gitlab do puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" - puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".green : "no"}" - puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".green : "no"}" - puts "Omniauth Providers: #{omniauth_providers.map(&:magenta).join(', ')}" if Gitlab.config.omniauth.enabled + puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}" + puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}" + puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled @@ -60,8 +60,8 @@ namespace :gitlab do end puts "" - puts "GitLab Shell".yellow - puts "Version:\t#{gitlab_shell_version || "unknown".red}" + puts "GitLab Shell".color(:yellow) + puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 4cbccf2ca89..05fcb8e3da5 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -14,12 +14,12 @@ namespace :gitlab do puts "" end - Rake::Task["db:setup"].invoke + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) exit 1 end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd61632e557..b1648a4602a 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -118,12 +118,12 @@ namespace :gitlab do puts "" unless $?.success? - puts "Failed to add keys...".red + puts "Failed to add keys...".color(:red) exit 1 end rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) exit 1 end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d33b5b31e18..d0c019044b7 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,7 +2,7 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -String.disable_colorization = true unless STDOUT.isatty +require 'rainbow/ext/string' # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] @@ -14,7 +14,7 @@ namespace :gitlab do # Returns "yes" the user chose to continue # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".blue, %w{yes no}) + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) raise Gitlab::TaskAbortedByUserError unless answer == "yes" end @@ -98,10 +98,10 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts " Warning ".colorize(:black).on_yellow - puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.magenta}." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." puts "" end @warned_user_not_gitlab = true diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 9196677a017..fc0ccc726ed 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -6,17 +6,17 @@ namespace :gitlab do count = scope.count if count > 0 - puts "This will disable 2FA for #{count.to_s.red} users..." + puts "This will disable 2FA for #{count.to_s.color(:red)} users..." begin ask_to_continue scope.find_each(&:disable_two_factor!) - puts "Successfully disabled 2FA for #{count} users.".green + puts "Successfully disabled 2FA for #{count} users.".color(:green) rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) end else - puts "There are currently no users with 2FA enabled.".yellow + puts "There are currently no users with 2FA enabled.".color(:yellow) end end end diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake index 9b636f12d9f..3bd10b0208b 100644 --- a/lib/tasks/gitlab/update_commit_count.rake +++ b/lib/tasks/gitlab/update_commit_count.rake @@ -6,15 +6,15 @@ namespace :gitlab do ask_to_continue unless ENV['force'] == 'yes' projects.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.yellow} ... " + print "#{project.name_with_namespace.color(:yellow)} ... " unless project.repo_exists? - puts "skipping, because the repo is empty".magenta + puts "skipping, because the repo is empty".color(:magenta) next end project.update_commit_count - puts project.commit_count.to_s.green + puts project.commit_count.to_s.color(:green) end end end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake new file mode 100644 index 00000000000..4fd48cccb1d --- /dev/null +++ b/lib/tasks/gitlab/update_gitignore.rake @@ -0,0 +1,46 @@ +namespace :gitlab do + desc "GitLab | Update gitignore" + task :update_gitignore do + unless clone_gitignores + puts "Cloning the gitignores failed".color(:red) + return + end + + remove_unneeded_files(gitignore_directory) + remove_unneeded_files(global_directory) + + puts "Done".color(:green) + end + + def clone_gitignores + FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) + FileUtils.cd vendor_directory + + system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') + end + + # Retain only certain files: + # - The LICENSE, because we have to + # - The sub dir global + # - The gitignores themself + # - Dir.entires returns also the entries '.' and '..' + def remove_unneeded_files(path) + Dir.foreach(path) do |file| + FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ + end + end + + private + + def vendor_directory + Rails.root.join('vendor') + end + + def gitignore_directory + File.join(vendor_directory, 'gitignore') + end + + def global_directory + File.join(gitignore_directory, 'Global') + end +end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index cc0f668474e..f467cc0ee29 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -12,9 +12,9 @@ namespace :gitlab do print "- #{project.name} ... " web_hook = project.hooks.new(url: web_hook_url) if web_hook.save - puts "added".green + puts "added".color(:green) else - print "failed".red + print "failed".color(:red) puts " [#{web_hook.errors.full_messages.to_sentence}]" end end @@ -57,7 +57,7 @@ namespace :gitlab do if namespace Project.in_namespace(namespace.id) else - puts "Namespace not found: #{namespace_path}".red + puts "Namespace not found: #{namespace_path}".color(:red) exit 2 end end diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index d258c6fd08d..4f2486157b7 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -1,6 +1,6 @@ desc "GitLab | Build internal ids for issues and merge requests" task migrate_iids: :environment do - puts 'Issues'.yellow + puts 'Issues'.color(:yellow) Issue.where(iid: nil).find_each(batch_size: 100) do |issue| begin issue.set_iid @@ -15,7 +15,7 @@ task migrate_iids: :environment do end puts 'done' - puts 'Merge Requests'.yellow + puts 'Merge Requests'.color(:yellow) MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr| begin mr.set_iid @@ -30,7 +30,7 @@ task migrate_iids: :environment do end puts 'done' - puts 'Milestones'.yellow + puts 'Milestones'.color(:yellow) Milestone.where(iid: nil).find_each(batch_size: 100) do |m| begin m.set_iid diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index ddfaf5d51f2..78ffccc9d06 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,4 +1,5 @@ unless Rails.env.production? require 'rubocop/rake_task' + RuboCop::RakeTask.new end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 01d23b89bb7..da255f5464b 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -52,7 +52,7 @@ def run_spinach_tests(tags) tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) puts '' - puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red + puts "Spinach tests for #{tags}: Retrying tests... #{tests}".color(:red) puts '' sleep(3) success = run_spinach_command(tests) |