diff options
| author | James Lopez <james@jameslopez.es> | 2016-06-13 14:00:19 +0200 |
|---|---|---|
| committer | James Lopez <james@jameslopez.es> | 2016-06-13 14:00:19 +0200 |
| commit | e83fc684a583140ce30644a7062f8470880fdf43 (patch) | |
| tree | 6c2a2f52da54d58def135757fcb5f14db8dd8a20 /lib | |
| parent | f70c000924c82bc579310fe784a6df159d15618b (diff) | |
| parent | 6f0c503397e184d2a68eec8d4285288abc29ce9a (diff) | |
| download | gitlab-ce-e83fc684a583140ce30644a7062f8470880fdf43.tar.gz | |
Merge branches 'feature/project-export-ui-experimental' and 'feature/project-import' of gitlab.com:gitlab-org/gitlab-ce into feature/project-export-ui-experimental
# Conflicts:
# Gemfile.lock
# app/helpers/todos_helper.rb
# app/models/todo.rb
# app/views/projects/edit.html.haml
# lib/gitlab/import_export/import_service.rb
Diffstat (limited to 'lib')
164 files changed, 3189 insertions, 1514 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index cc1004f8005..6cd909f6115 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,5 +1,3 @@ -Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} - module API class API < Grape::API include APIGuard @@ -25,38 +23,41 @@ module API format :json content_type :txt, "text/plain" - helpers Helpers - - mount Groups - mount GroupMembers - mount Users - mount Projects - mount Repositories - mount Issues - mount Milestones - mount Session - mount MergeRequests - mount Notes - mount Internal - mount SystemHooks - mount ProjectSnippets - mount ProjectMembers - mount DeployKeys - mount ProjectHooks - mount Services - mount Files - mount Commits - mount CommitStatus - mount Namespaces - mount Branches - mount Labels - mount Settings - mount Keys - mount Tags - mount Triggers - mount Builds - mount Variables - mount Runners - mount Licenses + # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::API::Helpers + + mount ::API::Groups + mount ::API::GroupMembers + mount ::API::Users + mount ::API::Projects + mount ::API::Repositories + mount ::API::Issues + mount ::API::Milestones + mount ::API::Session + mount ::API::MergeRequests + mount ::API::Notes + mount ::API::Internal + mount ::API::SystemHooks + mount ::API::ProjectSnippets + mount ::API::ProjectMembers + mount ::API::DeployKeys + mount ::API::ProjectHooks + mount ::API::Services + mount ::API::Files + mount ::API::Commits + mount ::API::CommitStatuses + mount ::API::Namespaces + mount ::API::Branches + mount ::API::Labels + mount ::API::Settings + mount ::API::Keys + mount ::API::Tags + mount ::API::Triggers + mount ::API::Builds + mount ::API::Variables + mount ::API::Runners + mount ::API::Licenses + mount ::API::Subscriptions + mount ::API::Gitignores end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9994fcefda..7e67edb203a 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -2,171 +2,175 @@ require 'rack/oauth2' -module APIGuard - extend ActiveSupport::Concern +module API + module APIGuard + extend ActiveSupport::Concern - included do |base| - # OAuth2 Resource Server Authentication - use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| - # The authenticator only fetches the raw token string + included do |base| + # OAuth2 Resource Server Authentication + use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| + # The authenticator only fetches the raw token string - # Must yield access token to store it in the env - request.access_token - end + # Must yield access token to store it in the env + request.access_token + end - helpers HelperMethods + helpers HelperMethods - install_error_responders(base) - end + install_error_responders(base) + end - # Helper Methods for Grape Endpoint - module HelperMethods - # Invokes the doorkeeper guard. - # - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it raises TokenNotFoundError. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + # Helper Methods for Grape Endpoint + module HelperMethods + # Invokes the doorkeeper guard. + # + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it raises TokenNotFoundError. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def doorkeeper_guard!(scopes: []) + if (access_token = find_access_token).nil? + raise TokenNotFoundError + + else + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + def doorkeeper_guard(scopes: []) + if access_token = find_access_token + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def current_user - @current_user - end + def current_user + @current_user + end - private - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) - end + private - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end + def find_access_token + @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + end - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end - end + def doorkeeper_request + @doorkeeper_request ||= ActionDispatch::Request.new(env) + end - module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes + def validate_access_token(access_token, scopes) + Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) end end - private - def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + module ClassMethods + # Installs the doorkeeper guard on the whole Grape API endpoint. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def guard_all!(scopes: []) + before do + guard! scopes: scopes + end + end - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler - end + private - def oauth2_bearer_token_error_handler - Proc.new do |e| - response = - case e - when MissingTokenError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - - when TokenNotFoundError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Bad Access Token.") - - when ExpiredError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token is expired. You can either do re-authorization or token refresh.") - - when RevokedError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token was revoked. You have to re-authorize from the user.") - - when InsufficientScopeError - # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) - # does not include WWW-Authenticate header, which breaks the standard. - Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( - :insufficient_scope, - Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], - { scope: e.scopes }) - end + def install_error_responders(base) + error_classes = [ MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] - response.finish + base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + end + + def oauth2_bearer_token_error_handler + Proc.new do |e| + response = + case e + when MissingTokenError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new + + when TokenNotFoundError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Bad Access Token.") + + when ExpiredError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is expired. You can either do re-authorization or token refresh.") + + when RevokedError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token was revoked. You have to re-authorize from the user.") + + when InsufficientScopeError + # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) + # does not include WWW-Authenticate header, which breaks the standard. + Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( + :insufficient_scope, + Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], + { scope: e.scopes }) + end + + response.finish + end end end - end - # - # Exceptions - # + # + # Exceptions + # - class MissingTokenError < StandardError; end + class MissingTokenError < StandardError; end - class TokenNotFoundError < StandardError; end + class TokenNotFoundError < StandardError; end - class ExpiredError < StandardError; end + class ExpiredError < StandardError; end - class RevokedError < StandardError; end + class RevokedError < StandardError; end - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes + class InsufficientScopeError < StandardError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes + end end end end diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 2b104f90aa7..0ff8fa74a84 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -33,7 +33,7 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.ci_commits.find_by_sha(params[:sha]) + commit = user_project.pipelines.find_by_sha(params[:sha]) return not_found! unless commit builds = commit.builds.order('id DESC') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 7388ed2f4ea..323a7086890 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -2,7 +2,7 @@ require 'mime/types' module API # Project commit statuses API - class CommitStatus < Grape::API + class CommitStatuses < Grape::API resource :projects do before { authenticate! } @@ -22,8 +22,8 @@ module API not_found!('Commit') unless user_project.commit(params[:sha]) - ci_commits = user_project.ci_commits.where(sha: params[:sha]) - statuses = ::CommitStatus.where(commit: ci_commits) + pipelines = user_project.pipelines.where(sha: params[:sha]) + statuses = ::CommitStatus.where(pipeline: pipelines) statuses = statuses.latest unless parse_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? statuses = statuses.where(stage: params[:stage]) if params[:stage].present? @@ -50,7 +50,7 @@ module API commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit - # Since the CommitStatus is attached to Ci::Commit (in the future Pipeline) + # Since the CommitStatus is attached to Ci::Pipeline (in the future Pipeline) # We need to always have the pipeline object # To have a valid pipeline object that can be attached to specific MR # Other CI service needs to send `ref` @@ -64,11 +64,11 @@ module API ref = branches.first end - ci_commit = @project.ensure_ci_commit(commit.sha, ref) + pipeline = @project.ensure_pipeline(commit.sha, ref) name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, commit: ci_commit, user: current_user) + status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) + status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) status.update(attrs) case params[:state].to_s diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 93a3a5ce089..4a11c8e3620 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -107,6 +107,8 @@ module API break if opts[:line_code] end + + opts[:type] = LegacyDiffNote.name if opts[:line_code] end note = ::Notes::CreateService.new(user_project, current_user, opts).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 716ca6f7ed9..14370ac218d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -30,7 +30,7 @@ module API expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project - expose :two_factor_enabled + expose :two_factor_enabled?, as: :two_factor_enabled expose :external end @@ -66,7 +66,8 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :created_at, :last_activity_at expose :shared_runners_enabled expose :creator_id expose :namespace @@ -174,11 +175,18 @@ module API expose :subscribed do |issue, options| issue.subscribed?(options[:current_user]) end + expose :user_notes_count + expose :upvotes, :downvotes + end + + class ExternalIssue < Grape::Entity + expose :title + expose :id end class MergeRequest < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels @@ -187,10 +195,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status - expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end + expose :user_notes_count end class MergeRequestChanges < MergeRequest @@ -216,8 +224,8 @@ module API expose :system?, as: :system expose :noteable_id, :noteable_type # upvote? and downvote? are deprecated, always return false - expose :upvote?, as: :upvote - expose :downvote?, as: :downvote + expose(:upvote?) { |note| false } + expose(:downvote?) { |note| false } end class MRNote < Grape::Entity @@ -227,9 +235,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_name } - expose(:line) { |note| note.diff_new_line } - expose(:line_type) { |note| note.diff_line_type } + expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } + expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } + expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } expose :author, using: Entities::UserBasic expose :created_at end @@ -307,6 +315,10 @@ module API class Label < Grape::Entity expose :name, :color, :description expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + + expose :subscribed do |label, options| + label.subscribed?(options[:current_user]) + end end class Compare < Grape::Entity @@ -344,6 +356,7 @@ module API expose :signin_enabled expose :gravatar_enabled expose :sign_in_text + expose :after_sign_up_text expose :created_at expose :updated_at expose :home_page_url @@ -357,6 +370,7 @@ module API expose :restricted_signup_domains expose :user_oauth_applications expose :after_sign_out_path + expose :container_registry_token_expire_delay end class Release < Grape::Entity @@ -403,6 +417,7 @@ module API class RunnerDetails < Runner expose :tag_list + expose :run_untagged expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -451,5 +466,13 @@ module API expose(:limitations) { |license| license.meta['limitations'] } expose :content end + + class GitignoresList < Grape::Entity + expose :name + end + + class Gitignore < Grape::Entity + expose :name, :content + end end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb new file mode 100644 index 00000000000..270c9501dd2 --- /dev/null +++ b/lib/api/gitignores.rb @@ -0,0 +1,29 @@ +module API + class Gitignores < Grape::API + + # Get the list of the available gitignore templates + # + # Example Request: + # GET /gitignores + get 'gitignores' do + present Gitlab::Gitignore.all, with: Entities::GitignoresList + end + + # Get the text for a specific gitignore + # + # Parameters: + # name (required) - The name of a license + # + # Example Request: + # GET /gitignores/Elixir + # + get 'gitignores/:name' do + required_attributes! [:name] + + gitignore = Gitlab::Gitignore.find(params[:name]) + not_found!('.gitignore') unless gitignore + + present gitignore, with: Entities::Gitignore + end + end +end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 91e420832f3..9d8b8d737a9 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -95,8 +95,7 @@ module API # GET /groups/:id/projects get ":id/projects" do group = find_group(params[:id]) - projects = group.projects - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects present projects, with: Entities::Project end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 40c967453fb..de5959e3aae 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,7 +2,7 @@ module API module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER ="HTTP_SUDO" + SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo def parse_boolean(value) @@ -29,7 +29,7 @@ module API @current_user end - def sudo_identifier() + def sudo_identifier identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers @@ -95,6 +95,17 @@ module API end end + def find_project_label(id) + label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label || not_found!('Label') + end + + def find_project_issue(id) + issue = user_project.issues.find(id) + not_found! unless can?(current_user, :read_issue, issue) + issue + end + def paginate(relation) relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| add_pagination_headers(data) @@ -397,5 +408,23 @@ module API error!(errors[:access_level], 422) if errors[:access_level].any? not_found!(errors) end + + def send_git_blob(repository, blob) + env['api.format'] = :txt + content_type 'text/plain' + header(*Gitlab::Workhorse.send_git_blob(repository, blob)) + end + + def send_git_archive(repository, ref:, format:) + header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) + end + + def issue_entity(project) + if project.has_external_issue_tracker? + Entities::ExternalIssue + else + Entities::Issue + end + end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 40928749481..4c43257c48a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -51,7 +51,7 @@ module API # GET /issues?labels=foo,bar # GET /issues?labels=foo,bar&state=opened get do - issues = current_user.issues + issues = current_user.issues.inc_notes_with_associations issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues.reorder(issuable_order_by => issuable_sort) @@ -82,7 +82,7 @@ module API # GET /projects/:id/issues?milestone=1.0.0&state=closed # GET /issues?iid=42 get ":id/issues" do - issues = user_project.issues.visible_to_user(current_user) + issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user) issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? @@ -103,8 +103,7 @@ module API # Example Request: # GET /projects/:id/issues/:issue_id get ":id/issues/:issue_id" do - @issue = user_project.issues.find(params[:issue_id]) - not_found! unless can?(current_user, :read_issue, @issue) + @issue = find_project_issue(params[:issue_id]) present @issue, with: Entities::Issue, current_user: current_user end @@ -234,42 +233,6 @@ module API authorize!(:destroy_issue, issue) issue.destroy end - - # Subscribes to a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # POST /projects/:id/issues/:issue_id/subscription - post ':id/issues/:issue_id/subscription' do - issue = user_project.issues.find(params[:issue_id]) - - if issue.subscribed?(current_user) - not_modified! - else - issue.toggle_subscription(current_user) - present issue, with: Entities::Issue, current_user: current_user - end - end - - # Unsubscribes from a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # DELETE /projects/:id/issues/:issue_id/subscription - delete ':id/issues/:issue_id/subscription' do - issue = user_project.issues.find(params[:issue_id]) - - if issue.subscribed?(current_user) - issue.unsubscribe(current_user) - present issue, with: Entities::Issue, current_user: current_user - else - not_modified! - end - end end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 4af6bef0fa7..c806829d69e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -11,7 +11,7 @@ module API # Example Request: # GET /projects/:id/labels get ':id/labels' do - present user_project.labels, with: Entities::Label + present user_project.labels, with: Entities::Label, current_user: current_user end # Creates a new label @@ -36,7 +36,7 @@ module API label = user_project.labels.create(attrs) if label.valid? - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end @@ -90,7 +90,7 @@ module API attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) if label.update(attrs) - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb index 187d2c04703..be0e113fbcb 100644 --- a/lib/api/licenses.rb +++ b/lib/api/licenses.rb @@ -2,15 +2,15 @@ module API # Licenses API class Licenses < Grape::API PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze # Get the list of the available license templates # diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 7e78609ecb9..0e94efd4acd 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -41,7 +41,7 @@ module API # get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests + merge_requests = user_project.merge_requests.inc_notes_with_associations unless params[:iid].nil? merge_requests = filter_by_iid(merge_requests, params[:iid]) @@ -218,6 +218,7 @@ module API # merge_commit_message (optional) - Custom merge commit message # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds + # sha (optional) - When present, must have the HEAD SHA of the source branch # Example: # PUT /projects/:id/merge_requests/:merge_request_id/merge # @@ -227,18 +228,21 @@ module API # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged + not_allowed! unless merge_request.mergeable_state? - render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + + if params[:sha] && merge_request.source_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.source_sha}", 409) + end merge_params = { commit_message: params[:merge_commit_message], should_remove_source_branch: params[:should_remove_source_branch] } - if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? + if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). execute(merge_request) else @@ -325,43 +329,7 @@ module API get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: Entities::Issue, current_user: current_user - end - - # Subscribes to a merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of a merge request - # Example Request: - # POST /projects/:id/issues/:merge_request_id/subscription - post "#{path}/subscription" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - if merge_request.subscribed?(current_user) - not_modified! - else - merge_request.toggle_subscription(current_user) - present merge_request, with: Entities::MergeRequest, current_user: current_user - end - end - - # Unsubscribes from a merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of a merge request - # Example Request: - # DELETE /projects/:id/merge_requests/:merge_request_id/subscription - delete "#{path}/subscription" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - if merge_request.subscribed?(current_user) - merge_request.unsubscribe(current_user) - present merge_request, with: Entities::MergeRequest, current_user: current_user - else - not_modified! - end + present paginate(issues), with: issue_entity(user_project), current_user: current_user end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 71a53e6f0d6..d4fcfd3d4d3 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -19,20 +19,24 @@ module API # GET /projects/:id/issues/:noteable_id/notes # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) + + if can?(current_user, noteable_read_ability_name(@noteable), @noteable) + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note + else + not_found!("Notes") + end end # Get a single +noteable+ note @@ -45,13 +49,14 @@ module API # GET /projects/:id/issues/:noteable_id/notes/:note_id # GET /projects/:id/snippets/:noteable_id/notes/:note_id get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) @note = @noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) - if @note.cross_reference_not_visible_for?(current_user) - not_found!("Note") - else + if can_read_note present @note, with: Entities::Note + else + not_found!("Note") end end @@ -136,5 +141,11 @@ module API end end end + + helpers do + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore.downcase}".to_sym + end + end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index cc2c7a0c503..5a22d14988f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -44,7 +44,7 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.starred_projects + @projects = current_user.viewable_starred_projects @projects = filter_projects(@projects) @projects = paginate @projects present @projects, with: Entities::Project @@ -94,6 +94,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # namespace_id (optional) - defaults to user namespace # public (optional) - if true same as setting visibility_level = 20 @@ -112,6 +113,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :namespace_id, :public, @@ -143,6 +145,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) @@ -206,6 +209,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project @@ -222,6 +226,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :public, :visibility_level, diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 62161aadb9a..f55aceed92c 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -56,8 +56,7 @@ module API blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) not_found! "File" unless blob - content_type 'text/plain' - header *Gitlab::Workhorse.send_git_blob(repo, blob) + send_git_blob repo, blob end # Get a raw blob contents by blob sha @@ -80,10 +79,7 @@ module API not_found! 'Blob' unless blob - env['api.format'] = :txt - - content_type blob.mime_type - header *Gitlab::Workhorse.send_git_blob(repo, blob) + send_git_blob repo, blob end # Get a an archive of the repository @@ -98,7 +94,7 @@ module API authorize! :download_code, user_project begin - header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) + send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue not_found!('File') end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 8ec91485b26..4faba9dc87b 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list] + attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else diff --git a/lib/api/session.rb b/lib/api/session.rb index cc646895914..56e69b2366f 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -11,8 +11,7 @@ module API # Example Request: # POST /session post "/session" do - auth = Gitlab::Auth.new - user = auth.find(params[:email] || params[:login], params[:password]) + user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password]) return unauthorized! unless user present user, with: Entities::UserLogin diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb new file mode 100644 index 00000000000..c49e2a21b82 --- /dev/null +++ b/lib/api/subscriptions.rb @@ -0,0 +1,60 @@ +module API + class Subscriptions < Grape::API + before { authenticate! } + + subscribable_types = { + 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'issues' => proc { |id| find_project_issue(id) }, + 'labels' => proc { |id| find_project_label(id) }, + } + + resource :projects do + subscribable_types.each do |type, finder| + type_singularized = type.singularize + type_id_str = :"#{type_singularized}_id" + entity_class = Entities.const_get(type_singularized.camelcase) + + # Subscribe to a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # POST /projects/:id/labels/:subscribable_id/subscription + # POST /projects/:id/issues/:subscribable_id/subscription + # POST /projects/:id/merge_requests/:subscribable_id/subscription + post ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if resource.subscribed?(current_user) + not_modified! + else + resource.subscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + + # Unsubscribe from a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # DELETE /projects/:id/labels/:subscribable_id/subscription + # DELETE /projects/:id/issues/:subscribable_id/subscription + # DELETE /projects/:id/merge_requests/:subscribable_id/subscription + delete ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if !resource.subscribed?(current_user) + not_modified! + else + resource.unsubscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index ea6fa2dc8a8..8a376d3c2a3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -76,7 +76,7 @@ module API required_attributes! [:email, :password, :name, :username] attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) + confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) user.admin = admin unless admin.nil? user.skip_confirmation! unless confirm diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb deleted file mode 100644 index b1aecc2e671..00000000000 --- a/lib/award_emoji.rb +++ /dev/null @@ -1,84 +0,0 @@ -class AwardEmoji - CATEGORIES = { - other: "Other", - objects: "Objects", - places: "Places", - travel_places: "Travel", - emoticons: "Emoticons", - objects_symbols: "Symbols", - nature: "Nature", - celebration: "Celebration", - people: "People", - activity: "Activity", - flags: "Flags", - food_drink: "Food" - }.with_indifferent_access - - CATEGORY_ALIASES = { - symbols: "objects_symbols", - foods: "food_drink", - travel: "travel_places" - }.with_indifferent_access - - def self.normilize_emoji_name(name) - aliases[name] || name - end - - def self.emoji_by_category - unless @emoji_by_category - @emoji_by_category = Hash.new { |h, key| h[key] = [] } - - emojis.each do |emoji_name, data| - data["name"] = emoji_name - - # Skip Fitzpatrick(tone) modifiers - next if data["category"] == "modifier" - - category = CATEGORY_ALIASES[data["category"]] || data["category"] - - @emoji_by_category[category] << data - end - - @emoji_by_category = @emoji_by_category.sort.to_h - end - - @emoji_by_category - end - - def self.emojis - @emojis ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) - JSON.parse(File.read(json_path)) - end - end - - def self.unicode - @unicode ||= emojis.map {|key, value| { key => emojis[key]["unicode"] } }.inject(:merge!) - end - - def self.aliases - @aliases ||= begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) - JSON.parse(File.read(json_path)) - end - end - - # Returns an Array of Emoji names and their asset URLs. - def self.urls - @urls ||= begin - path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - prefix = Gitlab::Application.config.assets.prefix - digest = Gitlab::Application.config.assets.digest - - JSON.parse(File.read(path)).map do |hash| - if digest - fname = "#{hash['unicode']}-#{hash['digest']}" - else - fname = hash['unicode'] - end - - { name: hash['name'], path: "#{prefix}/#{fname}.png" } - end - end - end -end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 67b2a64bd10..22319ec6623 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -86,9 +86,9 @@ module Backup def report_success(success) if success - $progress.puts '[DONE]'.green + $progress.puts '[DONE]'.color(:green) else - $progress.puts '[FAILED]'.red + $progress.puts '[FAILED]'.color(:red) end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..2ff3e3bdfb0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,5 +1,8 @@ module Backup class Manager + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] + FOLDERS_TO_BACKUP = %w[repositories db] + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -24,9 +27,9 @@ module Backup # Set file permissions on open to prevent chmod races. tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]} if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "creating archive #{tar_file} failed".red + puts "creating archive #{tar_file} failed".color(:red) abort 'Backup failed' end @@ -35,24 +38,22 @@ module Backup end def upload(tar_file) - remote_directory = Gitlab.config.backup.upload.remote_directory $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - $progress.puts "skipped".yellow + $progress.puts "skipped".color(:yellow) return end - connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) + directory = connect_to_remote_directory(connection_settings) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "uploading backup to #{remote_directory} failed".red + puts "uploading backup to #{remote_directory} failed".color(:red) abort 'Backup failed' end end @@ -64,9 +65,9 @@ module Backup next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "deleting tmp directory '#{dir}' failed".red + puts "deleting tmp directory '#{dir}' failed".color(:red) abort 'Backup failed' end end @@ -92,9 +93,9 @@ module Backup end end - $progress.puts "done. (#{removed} removed)".green + $progress.puts "done. (#{removed} removed)".color(:green) else - $progress.puts "skipping".yellow + $progress.puts "skipping".color(:yellow) end end @@ -121,20 +122,20 @@ module Backup $progress.print "Unpacking backup ... " unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red + puts "unpacking backup failed".color(:red) exit 1 else - $progress.puts "done".green + $progress.puts "done".color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".red - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".red - puts " Please switch to the following version and try again:".red - puts " version: #{settings[:gitlab_version]}".red + puts "GitLab version mismatch:".color(:red) + puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + puts " Please switch to the following version and try again:".color(:red) + puts " version: #{settings[:gitlab_version]}".color(:red) puts puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 @@ -147,21 +148,44 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) + settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item) end private + def connect_to_remote_directory(connection_settings) + connection = ::Fog::Storage.new(connection_settings) + + # We only attempt to create the directory for local backups. For AWS + # and other cloud providers, we cannot guarantee the user will have + # permission to create the bucket. + if connection.service == ::Fog::Storage::Local + connection.directories.create(key: remote_directory) + else + connection.directories.get(remote_directory) + end + end + + def remote_directory + Gitlab.config.backup.upload.remote_directory + end + def backup_contents folders_to_backup + archives_to_backup + ["backup_information.yml"] end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - %w{repositories db}.reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } + end + + def disabled_features + features = [] + features << 'registry' unless Gitlab.config.registry.enabled + features end def settings diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb new file mode 100644 index 00000000000..67fe0231087 --- /dev/null +++ b/lib/backup/registry.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Registry < Files + def initialize + super('registry', Settings.registry.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index a82a7e1f7bf..7b91215d50b 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -14,14 +14,14 @@ module Backup FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace if project.empty_repo? - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".green + $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".red + puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" puts output abort 'Backup failed' @@ -33,14 +33,14 @@ module Backup if File.exists?(path_to_repo(wiki)) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? - $progress.puts " [SKIPPED]".cyan + $progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Backup failed' end @@ -71,9 +71,9 @@ module Backup end if system(*cmd, silent) - $progress.puts "[DONE]".green + $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".red + puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end @@ -90,21 +90,21 @@ module Backup cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) if system(*cmd, silent) - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end end end - $progress.print 'Put GitLab hooks in repositories dirs'.yellow + $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" if system(cmd) - $progress.puts " [DONE]".green + $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".red + puts " [FAILED]".color(:red) puts "failed: #{cmd}" end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index b8962379cb5..db95d7c908b 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.data_reference - @data_reference ||= "data-#{object_name.dasherize}" - end - def self.object_class_title @object_title ||= object_class.name.titleize end @@ -45,10 +41,6 @@ module Banzai end end - def self.referenced_by(node) - { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } - end - def object_class self.class.object_class end @@ -236,7 +228,9 @@ module Banzai if cache.key?(key) cache[key] else - cache[key] = yield + value = yield + cache[key] = value if key.present? + value end end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index b469ea0f626..bbb88c979cc 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit_range + def self.object_class CommitRange end @@ -14,34 +16,18 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit-range") - range = find_object(project, id) - - return unless range - - { commit_range: range } - end - def initialize(*args) super @commit_map = {} end - def self.find_object(project, id) + def find_object(project, id) range = CommitRange.new(id, project) range.valid_commits? ? range : nil end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(range, project) h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index bd88207326c..2ce1816672b 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit + def self.object_class Commit end @@ -14,28 +16,12 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit") - commit = find_object(project, id) - - return unless commit - - { commit: commit } - end - - def self.find_object(project, id) + def find_object(project, id) if project && project.valid_repo? project.commit(id) end end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(commit, project) h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 37344b90576..eaa702952cc 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # References are ignored if the project doesn't use an external issue # tracker. class ExternalIssueReferenceFilter < ReferenceFilter + self.reference_type = :external_issue + # Public: Find `JIRA-123` issue references in text # # ExternalIssueReferenceFilter.references_in(text) do |match, issue| @@ -21,18 +23,6 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-external-issue") - external_issue = ExternalIssue.new(id, project) - - return unless external_issue - - { external_issue: external_issue } - end - def call # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 38c4219518e..f73ecfc9418 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -15,6 +15,7 @@ module Banzai next if link.start_with?(internal_url) node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') end doc diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb new file mode 100644 index 00000000000..beb21b19ab3 --- /dev/null +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -0,0 +1,26 @@ +module Banzai + module Filter + class InlineDiffFilter < HTML::Pipeline::Filter + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + + def call + search_text_nodes(doc).each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + content = node.to_html + html_content = inline_diff_filter(content) + + next if content == html_content + + node.replace(html_content) + end + doc + end + + def inline_diff_filter(text) + html_content = text.gsub(/(?:\[\-(.*?)\-\]|\{\-(.*?)\-\})/, '<span class="idiff left right deletion">\1\2</span>') + html_content.gsub(/(?:\[\+(.*?)\+\]|\{\+(.*?)\+\})/, '<span class="idiff left right addition">\1\2</span>') + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2732e0b5145..2496e704002 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -5,15 +5,12 @@ module Banzai # # This filter supports cross-project references. class IssueReferenceFilter < AbstractReferenceFilter + self.reference_type = :issue + def self.object_class Issue end - def self.user_can_see_reference?(user, node, context) - issue = Issue.find(node.attr('data-issue')) rescue nil - Ability.abilities.allowed?(user, :read_issue, issue) - end - def find_object(project, id) project.get_issue(id) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8488a493b55..e4d3f87d0aa 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -2,6 +2,8 @@ module Banzai module Filter # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter + self.reference_type = :label + def self.object_class Label end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index cad38a51851..ac5216d9cfb 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class MergeRequestReferenceFilter < AbstractReferenceFilter + self.reference_type = :merge_request + def self.object_class MergeRequest end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 4cb82178024..ca686c87d97 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -2,6 +2,8 @@ module Banzai module Filter # HTML filter that replaces milestone references with links. class MilestoneReferenceFilter < AbstractReferenceFilter + self.reference_type = :milestone + def self.object_class Milestone end @@ -10,11 +12,53 @@ module Banzai project.milestones.find_by(iid: id) end - def url_for_object(issue, project) + def references_in(text, pattern = Milestone.reference_pattern) + # We'll handle here the references that follow the `reference_pattern`. + # Other patterns (for example, the link pattern) are handled by the + # default implementation. + return super(text, pattern) if pattern != Milestone.reference_pattern + + text.gsub(pattern) do |match| + milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + + if milestone + yield match, milestone.iid, $~[:project], $~ + else + match + end + end + end + + def find_milestone(project_ref, milestone_id, milestone_name) + project = project_from_ref(project_ref) + return unless project + + milestone_params = milestone_params(milestone_id, milestone_name) + project.milestones.find_by(milestone_params) + end + + def milestone_params(iid, name) + if name + { name: name.tr('"', '') } + else + { iid: iid.to_i } + end + end + + def url_for_object(milestone, project) h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end + + def object_link_text(object, matches) + if context[:project] == object.project + super + else + "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>". + html_safe + end + end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index e589b5df6ec..c753a84a20d 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,8 +7,11 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Querying.css(doc, 'a.gfm').each do |node| - unless user_can_see_reference?(node) + nodes = Querying.css(doc, 'a.gfm[data-reference-type]') + visible = nodes_visible_to_user(nodes) + + nodes.each do |node| + unless visible.include?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text @@ -21,20 +24,30 @@ module Banzai private - def user_can_see_reference?(node) - if node.has_attribute?('data-reference-filter') - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, current_user) - reference_filter.user_can_see_reference?(current_user, node, context) - else - true + visible.merge(parser.nodes_visible_to_user(current_user, nodes)) end + + visible end def current_user context[:current_user] end + + def project + context[:project] + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 31386cf851c..2d6f34c9cd8 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -8,24 +8,8 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-project') - project_id = node.attr('data-project').to_i - return true if project_id == context[:project].try(:id) - - project = Project.find(project_id) rescue nil - Ability.abilities.allowed?(user, :read_project, project) - else - true - end - end - - def self.user_can_reference?(user, node, context) - true - end - - def self.referenced_by(node) - raise NotImplementedError, "#{self} does not implement #{__method__}" + class << self + attr_accessor :reference_type end # Returns a data attribute String to attach to a reference link @@ -43,7 +27,9 @@ module Banzai # # Returns a String def data_attribute(attributes = {}) - attributes[:reference_filter] = self.class.name.demodulize + attributes = attributes.reject { |_, v| v.nil? } + + attributes[:reference_type] = self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end @@ -82,6 +68,8 @@ module Banzai # by `ignore_ancestor_query`. Link tags are not processed if they have a # "gfm" class or the "href" attribute is empty. def each_node + return to_enum(__method__) unless block_given? + query = %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] | descendant-or-self::a[ not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") @@ -92,6 +80,11 @@ module Banzai end end + # Returns an Array containing all HTML nodes. + def nodes + @nodes ||= each_node.to_a + end + # Yields the link's URL and text whenever the node is a valid <a> tag. def yield_valid_link(node) link = CGI.unescape(node.attr('href').to_s) diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb deleted file mode 100644 index 96fdb06304e..00000000000 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ /dev/null @@ -1,65 +0,0 @@ -module Banzai - module Filter - # HTML filter that gathers all referenced records that the current user has - # permission to view. - # - # Expected to be run in its own post-processing pipeline. - # - class ReferenceGathererFilter < HTML::Pipeline::Filter - def initialize(*) - super - - result[:references] ||= Hash.new { |hash, type| hash[type] = [] } - end - - def call - Querying.css(doc, 'a.gfm').each do |node| - gather_references(node) - end - - load_lazy_references unless ReferenceExtractor.lazy? - - doc - end - - private - - def gather_references(node) - return unless node.has_attribute?('data-reference-filter') - - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) - - return if context[:reference_filter] && reference_filter != context[:reference_filter] - - return if author && !reference_filter.user_can_reference?(author, node, context) - - return unless reference_filter.user_can_see_reference?(current_user, node, context) - - references = reference_filter.referenced_by(node) - return unless references - - references.each do |type, values| - Array.wrap(values).each do |value| - result[:references][type] << value - end - end - end - - def load_lazy_references - refs = result[:references] - refs.each do |type, values| - refs[type] = ReferenceExtractor.lazily(values) - end - end - - def current_user - context[:current_user] - end - - def author - context[:author] - end - end - end -end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 42dbab9d27e..ca80aac5a08 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -63,7 +63,7 @@ module Banzai begin uri = Addressable::URI.parse(node['href']) - uri.scheme.strip! if uri.scheme + uri.scheme = uri.scheme.strip.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index d507eb5ebe1..212a0bbf2a0 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter + self.reference_type = :snippet + def self.object_class Snippet end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 7edfe5ade2d..c0f503c9af3 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,6 +8,8 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call + return doc unless project + doc.search('a').each do |el| process_link_attr el.attribute('href') end @@ -31,7 +33,11 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) + end + + def project + context[:project] end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index eea3af842b6..5b0a6d8541b 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter + self.reference_type = :user + # Public: Find `@user` user references in text # # UserReferenceFilter.references_in(text) do |match, username| @@ -21,50 +23,13 @@ module Banzai end end - def self.referenced_by(node) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - return unless group - - { user: group.users } - elsif node.has_attribute?('data-user') - { user: LazyReference.new(User, node.attr('data-user')) } - elsif node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return unless project - - { user: project.team.members.flatten } - end - end - - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - Ability.abilities.allowed?(user, :read_group, group) - else - super - end - end - - def self.user_can_reference?(user, node, context) - # Only team members can reference `@all` - if node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return false unless project - - user && project.team.member?(user) - else - super - end - end - def call return doc if project.nil? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ - each_node do |node| + nodes.each do |node| if text_node?(node) replace_text_when_pattern_matches(node, ref_pattern) do |content| user_link_filter(content) @@ -94,7 +59,7 @@ module Banzai self.class.references_in(text) do |match, username| if username == 'all' link_to_all(link_text: link_text) - elsif namespace = Namespace.find_by(path: username) + elsif namespace = namespaces[username] link_to_namespace(namespace, link_text: link_text) || match else match @@ -102,6 +67,31 @@ module Banzai end end + # Returns a Hash containing all Namespace objects for the username + # references in the current document. + # + # The keys of this Hash are the namespace paths, the values the + # corresponding Namespace objects. + def namespaces + @namespaces ||= + Namespace.where(path: usernames).each_with_object({}) do |row, hash| + hash[row.path] = row + end + end + + # Returns all usernames referenced in the current document. + def usernames + refs = Set.new + + nodes.each do |node| + node.to_html.scan(User.reference_pattern) do + refs << $~[:user] + end + end + + refs.to_a + end + private def urls @@ -114,9 +104,12 @@ module Banzai def link_to_all(link_text: nil) project = context[:project] + author = context[:author] + url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) - data = data_attribute(project: project.id) + + data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' link_tag(url, data, text) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 06d10c98501..37a2779d453 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -2,7 +2,8 @@ require 'uri' module Banzai module Filter - # HTML filter that "fixes" relative links to files in a repository. + # HTML filter that "fixes" links to pages/files in a wiki. + # Rewrite rules are documented in the `WikiPipeline` spec. # # Context options: # :project_wiki @@ -25,31 +26,15 @@ module Banzai end def process_link_attr(html_attr) - return if html_attr.blank? || file_reference?(html_attr) + return if html_attr.blank? - uri = URI(html_attr.value) - if uri.relative? && uri.path.present? - html_attr.value = rebuild_wiki_uri(uri).to_s - end + html_attr.value = apply_rewrite_rules(html_attr.value) rescue URI::Error # noop end - def rebuild_wiki_uri(uri) - uri.path = ::File.join(project_wiki_base_path, uri.path) - uri - end - - def file_reference?(html_attr) - !File.extname(html_attr.value).blank? - end - - def project_wiki - context[:project_wiki] - end - - def project_wiki_base_path - project_wiki && project_wiki.wiki_base_path + def apply_rewrite_rules(link_string) + Rewriter.new(link_string, wiki: context[:project_wiki], slug: context[:page_slug]).apply_rules end end end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb new file mode 100644 index 00000000000..2e2c8da311e --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -0,0 +1,40 @@ +module Banzai + module Filter + class WikiLinkFilter < HTML::Pipeline::Filter + class Rewriter + def initialize(link_string, wiki:, slug:) + @uri = Addressable::URI.parse(link_string) + @wiki_base_path = wiki && wiki.wiki_base_path + @slug = slug + end + + def apply_rules + apply_file_link_rules! + apply_hierarchical_link_rules! + apply_relative_link_rules! + @uri.to_s + end + + private + + # Of the form 'file.md' + def apply_file_link_rules! + @uri = Addressable::URI.join(@slug, @uri) if @uri.extname.present? + end + + # Of the form `./link`, `../link`, or similar + def apply_hierarchical_link_rules! + @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' + end + + # Any link _not_ of the form `http://example.com/` + def apply_relative_link_rules! + if @uri.relative? && @uri.path.present? + link = ::File.join(@wiki_base_path, @uri.path) + @uri = Addressable::URI.parse(link) + end + end + end + end + end +end diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb deleted file mode 100644 index 1095b4debc7..00000000000 --- a/lib/banzai/lazy_reference.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Banzai - class LazyReference - def self.load(refs) - lazy_references, values = refs.partition { |ref| ref.is_a?(self) } - - lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| - ids = refs.flat_map(&:ids) - klass.where(id: ids) - end - - values + lazy_values - end - - attr_reader :klass, :ids - - def initialize(klass, ids) - @klass = klass - @ids = Array.wrap(ids).map(&:to_i) - end - - def load - self.klass.where(id: self.ids) - end - end -end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index ed3cfd6b023..b27ecf3c923 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -23,7 +23,8 @@ module Banzai Filter::LabelReferenceFilter, Filter::MilestoneReferenceFilter, - Filter::TaskListFilter + Filter::TaskListFilter, + Filter::InlineDiffFilter ] end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb deleted file mode 100644 index 919998380e4..00000000000 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module Pipeline - class ReferenceExtractionPipeline < BasePipeline - def self.filters - FilterArray[ - Filter::ReferenceGathererFilter - ] - end - end - end -end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index f4079538ec5..bf366962aef 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,28 +1,6 @@ module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - class << self - LAZY_KEY = :banzai_reference_extractor_lazy - - def lazy? - Thread.current[LAZY_KEY] - end - - def lazily(values = nil, &block) - return (values || block.call).uniq if lazy? - - begin - Thread.current[LAZY_KEY] = true - - values ||= block.call - - Banzai::LazyReference.load(values.uniq).uniq - ensure - Thread.current[LAZY_KEY] = false - end - end - end - def initialize @texts = [] end @@ -31,23 +9,21 @@ module Banzai @texts << Renderer.render(text, context) end - def references(type, context = {}) - filter = Banzai::Filter["#{type}_reference"] + def references(type, project, current_user = nil) + processor = Banzai::ReferenceParser[type]. + new(project, current_user) + + processor.process(html_documents) + end - context.merge!( - pipeline: :reference_extraction, + private - # ReferenceGathererFilter - reference_filter: filter - ) + def html_documents + # This ensures that we don't memoize anything until we have a number of + # text blobs to parse. + return [] if @texts.empty? - self.class.lazily do - @texts.flat_map do |html| - text_context = context.dup - result = Renderer.render_result(html, text_context) - result[:references][type] - end.uniq - end + @html_documents ||= @texts.map { |html| Nokogiri::HTML.fragment(html) } end end end diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..557bec4316e --- /dev/null +++ b/lib/banzai/reference_parser.rb @@ -0,0 +1,14 @@ +module Banzai + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['issue'] + # + # This would return the `Banzai::ReferenceParser::IssueParser` class. + def self.[](name) + const_get("#{name.to_s.camelize}Parser") + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb new file mode 100644 index 00000000000..3d7b9c4a024 --- /dev/null +++ b/lib/banzai/reference_parser/base_parser.rb @@ -0,0 +1,204 @@ +module Banzai + module ReferenceParser + # Base class for reference parsing classes. + # + # Each parser should also specify its reference type by calling + # `self.reference_type = ...` in the body of the class. The value of this + # method should be a symbol such as `:issue` or `:merge_request`. For + # example: + # + # class IssueParser < BaseParser + # self.reference_type = :issue + # end + # + # The reference type is used to determine what nodes to pass to the + # `referenced_by` method. + # + # Parser classes should either implement the instance method + # `references_relation` or overwrite `referenced_by`. The + # `references_relation` method is supposed to return an + # ActiveRecord::Relation used as a base relation for retrieving the objects + # referenced in a set of HTML nodes. + # + # Each class can implement two additional methods: + # + # * `nodes_user_can_reference`: returns an Array of nodes the given user can + # refer to. + # * `nodes_visible_to_user`: returns an Array of nodes that are visible to + # the given user. + # + # You only need to overwrite these methods if you want to tweak who can see + # which references. For example, the IssueParser class defines its own + # `nodes_visible_to_user` method so it can ensure users can only see issues + # they have access to. + class BaseParser + class << self + attr_accessor :reference_type + end + + # Returns the attribute name containing the value for every object to be + # parsed by the current parser. + # + # For example, for a parser class that returns "Animal" objects this + # attribute would be "data-animal". + def self.data_attribute + @data_attribute ||= "data-#{reference_type.to_s.dasherize}" + end + + def initialize(project = nil, current_user = nil) + @project = project + @current_user = current_user + end + + # Returns all the nodes containing references that the user can refer to. + def nodes_user_can_reference(user, nodes) + nodes + end + + # Returns all the nodes that are visible to the given user. + def nodes_visible_to_user(user, nodes) + projects = lazy { projects_for_nodes(nodes) } + project_attr = 'data-project' + + nodes.select do |node| + if node.has_attribute?(project_attr) + node_id = node.attr(project_attr).to_i + + if project && project.id == node_id + true + else + can?(user, :read_project, projects[node_id]) + end + else + true + end + end + end + + # Returns an Array of objects referenced by any of the given HTML nodes. + def referenced_by(nodes) + ids = unique_attribute_values(nodes, self.class.data_attribute) + + references_relation.where(id: ids) + end + + # Returns the ActiveRecord::Relation to use for querying references in the + # DB. + def references_relation + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + # Returns a Hash containing attribute values per project ID. + # + # The returned Hash uses the following format: + # + # { project id => [value1, value2, ...] } + # + # nodes - An Array of HTML nodes to process. + # attribute - The name of the attribute (as a String) for which to gather + # values. + # + # Returns a Hash. + def gather_attributes_per_project(nodes, attribute) + per_project = Hash.new { |hash, key| hash[key] = Set.new } + + nodes.each do |node| + project_id = node.attr('data-project').to_i + id = node.attr(attribute) + + per_project[project_id] << id if id + end + + per_project + end + + # Returns a Hash containing objects for an attribute grouped per their + # IDs. + # + # The returned Hash uses the following format: + # + # { id value => row } + # + # nodes - An Array of HTML nodes to process. + # + # collection - The model or ActiveRecord relation to use for retrieving + # rows from the database. + # + # attribute - The name of the attribute containing the primary key values + # for every row. + # + # Returns a Hash. + def grouped_objects_for_nodes(nodes, collection, attribute) + return {} if nodes.empty? + + ids = unique_attribute_values(nodes, attribute) + + collection.where(id: ids).each_with_object({}) do |row, hash| + hash[row.id] = row + end + end + + # Returns an Array containing all unique values of an attribute of the + # given nodes. + def unique_attribute_values(nodes, attribute) + values = Set.new + + nodes.each do |node| + if node.has_attribute?(attribute) + values << node.attr(attribute) + end + end + + values.to_a + end + + # Processes the list of HTML documents and returns an Array containing all + # the references. + def process(documents) + type = self.class.reference_type + + nodes = documents.flat_map do |document| + Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a + end + + gather_references(nodes) + end + + # Gathers the references for the given HTML nodes. + def gather_references(nodes) + nodes = nodes_user_can_reference(current_user, nodes) + nodes = nodes_visible_to_user(current_user, nodes) + + referenced_by(nodes) + end + + # Returns a Hash containing the projects for a given list of HTML nodes. + # + # The returned Hash uses the following format: + # + # { project ID => project } + # + def projects_for_nodes(nodes) + @projects_for_nodes ||= + grouped_objects_for_nodes(nodes, Project, 'data-project') + end + + def can?(user, permission, subject) + Ability.abilities.allowed?(user, permission, subject) + end + + def find_projects_for_hash_keys(hash) + Project.where(id: hash.keys) + end + + private + + attr_reader :current_user, :project + + def lazy(&block) + Gitlab::Lazy.new(&block) + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb new file mode 100644 index 00000000000..0fee9d267de --- /dev/null +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -0,0 +1,34 @@ +module Banzai + module ReferenceParser + class CommitParser < BaseParser + self.reference_type = :commit + + def referenced_by(nodes) + commit_ids = commit_ids_per_project(nodes) + projects = find_projects_for_hash_keys(commit_ids) + + projects.flat_map do |project| + find_commits(project, commit_ids[project.id]) + end + end + + def commit_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_commits(project, ids) + commits = [] + + return commits unless project.valid_repo? + + ids.each do |id| + commit = project.commit(id) + + commits << commit if commit + end + + commits + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb new file mode 100644 index 00000000000..69d01f8db15 --- /dev/null +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -0,0 +1,38 @@ +module Banzai + module ReferenceParser + class CommitRangeParser < BaseParser + self.reference_type = :commit_range + + def referenced_by(nodes) + range_ids = commit_range_ids_per_project(nodes) + projects = find_projects_for_hash_keys(range_ids) + + projects.flat_map do |project| + find_ranges(project, range_ids[project.id]) + end + end + + def commit_range_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_ranges(project, range_ids) + ranges = [] + + range_ids.each do |id| + range = find_object(project, id) + + ranges << range if range + end + + ranges + end + + def find_object(project, id) + range = CommitRange.new(id, project) + + range.valid_commits? ? range : nil + end + end + end +end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb new file mode 100644 index 00000000000..a1264db2111 --- /dev/null +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class ExternalIssueParser < BaseParser + self.reference_type = :external_issue + + def referenced_by(nodes) + issue_ids = issue_ids_per_project(nodes) + projects = find_projects_for_hash_keys(issue_ids) + issues = [] + + projects.each do |project| + issue_ids[project.id].each do |id| + issues << ExternalIssue.new(id, project) + end + end + + issues + end + + def issue_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb new file mode 100644 index 00000000000..24076e3d9ec --- /dev/null +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -0,0 +1,40 @@ +module Banzai + module ReferenceParser + class IssueParser < BaseParser + self.reference_type = :issue + + def nodes_visible_to_user(user, nodes) + # It is not possible to check access rights for external issue trackers + return nodes if project && project.external_issue_tracker + + issues = issues_for_nodes(nodes) + + nodes.select do |node| + issue = issue_for_node(issues, node) + + issue ? can?(user, :read_issue, issue) : false + end + end + + def referenced_by(nodes) + issues = issues_for_nodes(nodes) + + nodes.map { |node| issue_for_node(issues, node) }.uniq + end + + def issues_for_nodes(nodes) + @issues_for_nodes ||= grouped_objects_for_nodes( + nodes, + Issue.all.includes(:author, :assignee, :project), + self.class.data_attribute + ) + end + + private + + def issue_for_node(issues, node) + issues[node.attr(self.class.data_attribute).to_i] + end + end + end +end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb new file mode 100644 index 00000000000..e5d1eb11d7f --- /dev/null +++ b/lib/banzai/reference_parser/label_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class LabelParser < BaseParser + self.reference_type = :label + + def references_relation + Label + end + end + end +end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb new file mode 100644 index 00000000000..c9a9ca79c09 --- /dev/null +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MergeRequestParser < BaseParser + self.reference_type = :merge_request + + def references_relation + MergeRequest.includes(:author, :assignee, :target_project) + end + end + end +end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb new file mode 100644 index 00000000000..a000ac61e5c --- /dev/null +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MilestoneParser < BaseParser + self.reference_type = :milestone + + def references_relation + Milestone + end + end + end +end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb new file mode 100644 index 00000000000..fa71b3c952a --- /dev/null +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class SnippetParser < BaseParser + self.reference_type = :snippet + + def references_relation + Snippet + end + end + end +end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb new file mode 100644 index 00000000000..a12b0d19560 --- /dev/null +++ b/lib/banzai/reference_parser/user_parser.rb @@ -0,0 +1,92 @@ +module Banzai + module ReferenceParser + class UserParser < BaseParser + self.reference_type = :user + + def referenced_by(nodes) + group_ids = [] + user_ids = [] + project_ids = [] + + nodes.each do |node| + if node.has_attribute?('data-group') + group_ids << node.attr('data-group').to_i + elsif node.has_attribute?(self.class.data_attribute) + user_ids << node.attr(self.class.data_attribute).to_i + elsif node.has_attribute?('data-project') + project_ids << node.attr('data-project').to_i + end + end + + find_users_for_groups(group_ids) | find_users(user_ids) | + find_users_for_projects(project_ids) + end + + def nodes_visible_to_user(user, nodes) + group_attr = 'data-group' + groups = lazy { grouped_objects_for_nodes(nodes, Group, group_attr) } + visible = [] + remaining = [] + + nodes.each do |node| + if node.has_attribute?(group_attr) + node_group = groups[node.attr(group_attr).to_i] + + if node_group && + can?(user, :read_group, node_group) + visible << node + end + # Remaining nodes will be processed by the parent class' + # implementation of this method. + else + remaining << node + end + end + + visible + super(current_user, remaining) + end + + def nodes_user_can_reference(current_user, nodes) + project_attr = 'data-project' + author_attr = 'data-author' + + projects = lazy { projects_for_nodes(nodes) } + users = lazy { grouped_objects_for_nodes(nodes, User, author_attr) } + + nodes.select do |node| + project_id = node.attr(project_attr) + user_id = node.attr(author_attr) + + if project && project_id && project.id == project_id.to_i + true + elsif project_id && user_id + project = projects[project_id.to_i] + user = users[user_id.to_i] + + project && user ? project.team.member?(user) : false + else + true + end + end + end + + def find_users(ids) + return [] if ids.empty? + + User.where(id: ids).to_a + end + + def find_users_for_groups(ids) + return [] if ids.empty? + + User.joins(:group_members).where(members: { source_id: ids }).to_a + end + + def find_users_for_projects(ids) + return [] if ids.empty? + + Project.where(id: ids).flat_map { |p| p.team.members.to_a } + end + end + end +end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index ac6d667cf8d..229050151d3 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -23,8 +23,8 @@ module Ci cross: 0x10, } - def self.convert(ansi) - Converter.new().convert(ansi) + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) end class Converter @@ -84,22 +84,38 @@ module Ci def on_107(s) set_bg_color(7, 'l') end def on_109(s) set_bg_color(9, 'l') end - def convert(ansi) - @out = "" - @n_open_tags = 0 - reset() + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] + + def convert(raw, new_state) + reset_state + restore_state(raw, new_state) if new_state.present? + + start = @offset + ansi = raw[@offset..-1] + + open_new_tag - s = StringScanner.new(ansi.gsub("<", "<")) - while(!s.eos?) + s = StringScanner.new(ansi) + until s.eos? if s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\n/) + @out << '<br>' else @out << s.scan(/./m) end + @offset += s.matched_size end close_open_tags() - @out + + { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } end def handle_sequence(s) @@ -121,6 +137,20 @@ module Ci evaluate_command_stack(commands) + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag css_classes = [] unless @fg_color.nil? @@ -138,20 +168,8 @@ module Ci css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - open_new_tag(css_classes) if css_classes.length > 0 - end + return if css_classes.empty? - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) - end - - evaluate_command_stack(stack) - end - - def open_new_tag(css_classes) @out << %{<span class="#{css_classes.join(' ')}">} @n_open_tags += 1 end @@ -163,6 +181,31 @@ module Ci end end + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(raw, new_state) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > raw.length + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) + end + end + def reset @fg_color = nil @bg_color = nil diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 353c4ddebf8..17bb99a2ae5 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -1,9 +1,7 @@ -Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} - module Ci module API class API < Grape::API - include APIGuard + include ::API::APIGuard version 'v1', using: :path rescue_from ActiveRecord::RecordNotFound do @@ -31,9 +29,9 @@ module Ci helpers ::API::Helpers helpers Gitlab::CurrentSettings - mount Builds - mount Runners - mount Triggers + mount ::Ci::API::Builds + mount ::Ci::API::Runners + mount ::Ci::API::Triggers end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b25e0e573a8..a902ced35d7 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -56,7 +56,7 @@ module Ci class TriggerRequest < Grape::Entity expose :id, :variables - expose :commit, using: Commit + expose :pipeline, using: Commit, as: :commit end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 192b1d18a51..0c41f22c7c5 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,20 +28,20 @@ module Ci post "register" do required_attributes! [:token] + attributes = { description: params[:description], + tag_list: params[:tag_list] } + + unless params[:run_untagged].nil? + attributes[:run_untagged] = params[:run_untagged] + end + runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create( - description: params[:description], - tag_list: params[:tag_list], - is_shared: true - ) + Ci::Runner.create(attributes.merge(is_shared: true)) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. - project.runners.create( - description: params[:description], - tag_list: params[:tag_list] - ) + project.runners.create(attributes) end return forbidden! unless runner diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index d53bdcbd0f2..5270108ef0f 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,11 +60,12 @@ module Ci class BuildTime < Chart def collect - commits = project.ci_commits.last(30) + commits = project.pipelines.last(30) commits.each do |commit| @labels << commit.short_sha - @build_times << (commit.duration / 60) + duration = commit.duration || 0 + @build_times << (duration / 60) end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 504d3df9d34..130f5b0892e 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -1,6 +1,6 @@ module Ci class GitlabCiYamlProcessor - class ValidationError < StandardError;end + class ValidationError < StandardError; end DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' @@ -12,18 +12,14 @@ module Ci attr_reader :before_script, :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = YAML.safe_load(config, [Symbol], [], true) + @config = Gitlab::Ci::Config.new(config).to_hash @path = path - unless @config.is_a? Hash - raise ValidationError, "YAML should be a hash" - end - - @config = @config.deep_symbolize_keys - initial_parsing validate! + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message end def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) @@ -265,7 +261,7 @@ module Ci end def validate_job_dependencies!(name, job) - if !validate_array_of_strings(job[:dependencies]) + unless validate_array_of_strings(job[:dependencies]) raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..4e20dc4f875 --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + delegate :registry, :client, to: :repository + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..4d726692f45 --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,61 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + @faraday.get("/v2/#{name}/tags/list").body + end + + def repository_manifest(name, reference) + @faraday.get("/v2/#{name}/manifests/#{reference}").body + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: /\bjson$/ + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..589f9f4380a --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,16 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..0e634f6b6ef --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,21 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @uri = uri + @path = options[:path] || default_path + @client = ContainerRegistry::Client.new(uri, options) + end + + def repository(name) + ContainerRegistry::Repository.new(self, name) + end + + private + + def default_path + @uri.sub(/^https?:\/\//, '') + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..0e4a7cb3cc9 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + delegate :client, to: :registry + + def initialize(registry, name) + @registry, @name = registry, name + end + + def path + [registry.path, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..43f8d6dc8c2 --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,77 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + delegate :registry, :client, to: :repository + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + + manifest[key] + end + + def digest + return @digest if defined?(@digest) + + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + + @config_blob = repository.blob(manifest['config']) + end + + def config + return unless config_blob + + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + + @layers = manifest['layers'].map do |layer| + repository.blob(layer) + end + end + + def total_size + return unless layers + + layers.map(&:size).sum + end + + def delete + return unless digest + + client.delete_repository_tag(repository.name, digest) + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f15b2cfd231..668d2fa41b3 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -27,7 +27,7 @@ class EventFilter @params = if params params.dup else - []#EventFilter.default_filter + [] # EventFilter.default_filter end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 7479e729db1..37f4c34054f 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,4 @@ -require 'gitlab/git' +require_dependency 'gitlab/git' module Gitlab def self.com? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 30509528b8b..076e2af7d38 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,17 +1,86 @@ module Gitlab - class Auth - def find(login, password) - user = User.by_login(login) - - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? - - Gitlab::LDAP::Authentication.login(login, password) - else - user if user.valid_password?(password) + module Auth + Result = Struct.new(:user, :type) + + class << self + def find(login, password, project:, ip:) + raise "Must provide an IP for rate limiting" if ip.nil? + + result = Result.new + + if valid_ci_request?(login, password, project) + result.type = :ci + elsif result.user = find_in_gitlab_or_ldap(login, password) + result.type = :gitlab_or_ldap + elsif result.user = oauth_access_token_check(login, password) + result.type = :oauth + end + + rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) + result + end + + def find_in_gitlab_or_ldap(login, password) + user = User.by_login(login) + + # If no user is found, or it's an LDAP server, try LDAP. + # LDAP users are only authenticated via LDAP + if user.nil? || user.ldap_user? + # Second chance - try LDAP authentication + return nil unless Gitlab::LDAP::Config.enabled? + + Gitlab::LDAP::Authentication.login(login, password) + else + user if user.valid_password?(password) + end + end + + def rate_limit!(ip, success:, login:) + rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) + return unless rate_limiter.enabled? + + if success + # Repeated login 'failures' are normal behavior for some Git clients so + # it is important to reset the ban counter once the client has proven + # they are not a 'bad guy'. + rate_limiter.reset! + else + # Register a login failure so that Rack::Attack can block the next + # request from this IP if needed. + rate_limiter.register_fail! + + if rate_limiter.banned? + Rails.logger.info "IP #{ip} failed to login " \ + "as #{login} but has been temporarily banned from Git auth" + end + end + end + + private + + def valid_ci_request?(login, password, project) + matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) + + return false unless project && matched_login.present? + + underscored_service = matched_login['service'].underscore + + if underscored_service == 'gitlab_ci' + project && project.valid_build_token?(password) + elsif Service.available_services_names.include?(underscored_service) + # We treat underscored_service as a trusted input because it is included + # in the Service.available_services_names whitelist. + service = project.public_send("#{underscored_service}_service") + + service && service.activated? && service.valid_token?(password) + end + end + + def oauth_access_token_check(login, password) + if login == "oauth2" && password.present? + token = Doorkeeper::AccessToken.by_token(password) + token && token.accessible? && User.find_by(id: token.resource_owner_id) + end end end end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb new file mode 100644 index 00000000000..1089bc9f89e --- /dev/null +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -0,0 +1,42 @@ +module Gitlab + module Auth + class IpRateLimiter + attr_reader :ip + + def initialize(ip) + @ip = ip + @banned = false + end + + def enabled? + config.enabled + end + + def reset! + Rack::Attack::Allow2Ban.reset(ip, config) + end + + def register_fail! + # Allow2Ban.filter will return false if this IP has not failed too often yet + @banned = Rack::Attack::Allow2Ban.filter(ip, config) do + # If we return false here, the failure for this IP is ignored by Allow2Ban + ip_can_be_banned? + end + end + + def banned? + @banned + end + + private + + def config + Gitlab.config.rack_attack.git_basic_auth + end + + def ip_can_be_banned? + config.ip_whitelist.exclude?(ip) + end + end + end +end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb new file mode 100644 index 00000000000..51b1df9ecbd --- /dev/null +++ b/lib/gitlab/award_emoji.rb @@ -0,0 +1,84 @@ +module Gitlab + class AwardEmoji + CATEGORIES = { + other: "Other", + objects: "Objects", + places: "Places", + travel_places: "Travel", + emoticons: "Emoticons", + objects_symbols: "Symbols", + nature: "Nature", + celebration: "Celebration", + people: "People", + activity: "Activity", + flags: "Flags", + food_drink: "Food" + }.with_indifferent_access + + CATEGORY_ALIASES = { + symbols: "objects_symbols", + foods: "food_drink", + travel: "travel_places" + }.with_indifferent_access + + def self.normalize_emoji_name(name) + aliases[name] || name + end + + def self.emoji_by_category + unless @emoji_by_category + @emoji_by_category = Hash.new { |h, key| h[key] = [] } + + emojis.each do |emoji_name, data| + data["name"] = emoji_name + + # Skip Fitzpatrick(tone) modifiers + next if data["category"] == "modifier" + + category = CATEGORY_ALIASES[data["category"]] || data["category"] + + @emoji_by_category[category] << data + end + + @emoji_by_category = @emoji_by_category.sort.to_h + end + + @emoji_by_category + end + + def self.emojis + @emojis ||= + begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) + JSON.parse(File.read(json_path)) + end + end + + def self.aliases + @aliases ||= + begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) + JSON.parse(File.read(json_path)) + end + end + + # Returns an Array of Emoji names and their asset URLs. + def self.urls + @urls ||= begin + path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + prefix = Gitlab::Application.config.assets.prefix + digest = Gitlab::Application.config.assets.digest + + JSON.parse(File.read(path)).map do |hash| + if digest + fname = "#{hash['unicode']}-#{hash['digest']}" + else + fname = hash['unicode'] + end + + { name: hash['name'], path: "#{prefix}/#{fname}.png" } + end + end + end + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index cdcaae8094c..9e09d2e118d 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -36,10 +36,7 @@ module Grack lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call return lfs_response unless lfs_response.nil? - if project && authorized_request? - # Tell gitlab-workhorse the request is OK, and what the GL_ID is - render_grack_auth_ok - elsif @user.nil? && !@ci + if @user.nil? && !@ci unauthorized else render_not_found @@ -98,7 +95,7 @@ module Grack end def authenticate_user(login, password) - user = Gitlab::Auth.new.find(login, password) + user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password) unless user user = oauth_access_token_check(login, password) @@ -141,36 +138,6 @@ module Grack user end - def authorized_request? - return true if @ci - - case git_cmd - when *Gitlab::GitAccess::DOWNLOAD_COMMANDS - if !Gitlab.config.gitlab_shell.upload_pack - false - elsif user - Gitlab::GitAccess.new(user, project).download_access_check.allowed? - elsif project.public? - # Allow clone/fetch for public projects - true - else - false - end - when *Gitlab::GitAccess::PUSH_COMMANDS - if !Gitlab.config.gitlab_shell.receive_pack - false - elsif user - # Skip user authorization on upload request. - # It will be done by the pre-receive hook in the repository. - true - else - false - end - else - false - end - end - def git_cmd if @request.get? @request.params['service'] @@ -197,24 +164,6 @@ module Grack end end - def render_grack_auth_ok - repo_path = - if @request.path_info =~ /^([\w\.\/-]+)\.wiki\.git/ - ProjectWiki.new(project).repository.path_to_repo - else - project.repository.path_to_repo - end - - [ - 200, - { "Content-Type" => "application/json" }, - [JSON.dump({ - 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), - 'RepoPath' => repo_path, - })] - ] - end - def render_not_found [404, { "Content-Type" => "text/plain" }, ["Not Found"]] end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 132f9cd1966..3e3986d6382 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -180,7 +180,7 @@ module Gitlab # exists?('gitlab/cookies.git') # def exists?(dir_name) - File.exists?(full_path(dir_name)) + File.exist?(full_path(dir_name)) end protected diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index 9b83292ef33..8d1ad62fae0 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -121,7 +121,7 @@ module Gitlab def get(url) response = api.get(url) - raise Unauthorized if (400..499).include?(response.code.to_i) + raise Unauthorized if (400..499).cover?(response.code.to_i) response end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 941f818b847..b90ef0b0fba 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["slug"], @@ -21,11 +21,8 @@ module Gitlab import_type: "bitbucket", import_source: "#{repo["owner"]}/#{repo["slug"]}", import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", + import_data: { credentials: { bb_session: session_data } } ).execute - - project.create_or_update_import_data(credentials: { bb_session: session_data }) - - project end end end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 34e949130da..9f45aefda0f 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -3,7 +3,7 @@ module Gitlab class << self def build(build) project = build.project - commit = build.commit + commit = build.pipeline user = build.user data = { diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index f2020c82d40..cd2e83b4c27 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab child_pattern = '[^/]*/?$' unless @opts[:recursive] match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ - until gz.eof? do + until gz.eof? begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb new file mode 100644 index 00000000000..ffe633d4b63 --- /dev/null +++ b/lib/gitlab/ci/config.rb @@ -0,0 +1,16 @@ +module Gitlab + module Ci + class Config + class LoaderError < StandardError; end + + def initialize(config) + loader = Loader.new(config) + @config = loader.load! + end + + def to_hash + @config + end + end + end +end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb new file mode 100644 index 00000000000..dbf6eb0edbe --- /dev/null +++ b/lib/gitlab/ci/config/loader.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + class Config + class Loader + class FormatError < StandardError; end + + def initialize(config) + @config = YAML.safe_load(config, [Symbol], [], true) + end + + def valid? + @config.is_a?(Hash) + end + + def load! + unless valid? + raise FormatError, 'Invalid configuration format' + end + + @config.deep_symbolize_keys + end + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 85583dce9ee..9dc2602867e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -19,7 +19,7 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..(Date.today + 1.day)).to_a + dates = (1.year.ago.to_date..Date.today).to_a dates.each do |date| date_id = date.to_time.to_i.to_s diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 688e780c13d..38c0f1aba47 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,18 +1,22 @@ module Gitlab module CurrentSettings def current_application_settings - key = :current_application_settings - - RequestStore.store[key] ||= begin - settings = nil + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - if connect_to_db? - settings = ::ApplicationSetting.current - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? - end + def ensure_application_settings! + settings = ::ApplicationSetting.cached - settings || fake_application_settings + if !settings && connect_to_db? + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end def fake_application_settings @@ -22,7 +26,10 @@ module Gitlab signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], gravatar_enabled: Settings.gravatar['enabled'], - sign_in_text: Settings.extra['sign_in_text'], + sign_in_text: nil, + after_sign_up_text: nil, + help_page_text: nil, + shared_runners_text: nil, restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], @@ -36,6 +43,7 @@ module Gitlab two_factor_grace_period: 48, akismet_enabled: false, repository_checks_enabled: true, + container_registry_token_expire_delay: 5, ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 6f9da69983a..04fa6a3a5de 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -5,17 +5,31 @@ module Gitlab end def self.mysql? - adapter_name.downcase == 'mysql2' + adapter_name.casecmp('mysql2').zero? end def self.postgresql? - adapter_name.downcase == 'postgresql' + adapter_name.casecmp('postgresql').zero? end def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end + def self.nulls_last_order(field, direction = 'ASC') + order = "#{field} #{direction}" + + if Gitlab::Database.postgresql? + order << ' NULLS LAST' + else + # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL + # columns. In the (default) ascending order, `0` comes first. + order.prepend("#{field} IS NULL, ") if direction == 'ASC' + end + + order + end + def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb new file mode 100644 index 00000000000..978c3f7896d --- /dev/null +++ b/lib/gitlab/database/migration_helpers.rb @@ -0,0 +1,142 @@ +module Gitlab + module Database + module MigrationHelpers + # Creates a new index, concurrently when supported + # + # On PostgreSQL this method creates an index concurrently, on MySQL this + # creates a regular index. + # + # Example: + # + # add_concurrent_index :users, :some_column + # + # See Rails' `add_index` for more info on the available arguments. + def add_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'add_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + end + + add_index(table_name, column_name, options) + end + + # Updates the value of a column in batches. + # + # This method updates the table in batches of 5% of the total row count. + # Any data inserted while running this method (or after it has finished + # running) is _not_ updated automatically. + # + # This method _only_ updates rows where the column's value is set to NULL. + # + # table - The name of the table. + # column - The name of the column to update. + # value - The value for the column. + def update_column_in_batches(table, column, value) + quoted_table = quote_table_name(table) + quoted_column = quote_column_name(column) + + ## + # Workaround for #17711 + # + # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` + # returns correct value (1), but `ActiveRecord::Migration.new.quote` + # returns incorrect value ('true'), which causes migrations to fail. + # + quoted_value = connection.quote(value) + processed = 0 + + total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). + to_hash. + first['count']. + to_i + + # Update in batches of 5% + batch_size = ((total / 100.0) * 5.0).ceil + + while processed < total + start_row = exec_query(%Q{ + SELECT id + FROM #{quoted_table} + ORDER BY id ASC + LIMIT 1 OFFSET #{processed} + }).to_hash.first + + stop_row = exec_query(%Q{ + SELECT id + FROM #{quoted_table} + ORDER BY id ASC + LIMIT 1 OFFSET #{processed + batch_size} + }).to_hash.first + + query = %Q{ + UPDATE #{quoted_table} + SET #{quoted_column} = #{quoted_value} + WHERE id >= #{start_row['id']} + } + + if stop_row + query += " AND id < #{stop_row['id']}" + end + + execute(query) + + processed += batch_size + end + end + + # Adds a column with a default value without locking an entire table. + # + # This method runs the following steps: + # + # 1. Add the column with a default value of NULL. + # 2. Update all existing rows in batches. + # 3. Change the default value of the column to the specified value. + # 4. Update any remaining rows. + # + # These steps ensure a column can be added to a large and commonly used + # table without locking the entire table for the duration of the table + # modification. + # + # table - The name of the table to update. + # column - The name of the column to add. + # type - The column type (e.g. `:integer`). + # default - The default value for the column. + # allow_null - When set to `true` the column will allow NULL values, the + # default is to not allow NULL values. + def add_column_with_default(table, column, type, default:, allow_null: false) + if transaction_open? + raise 'add_column_with_default can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + transaction do + add_column(table, column, type, default: nil) + + # Changing the default before the update ensures any newly inserted + # rows already use the proper default value. + change_column_default(table, column, default) + end + + begin + transaction do + update_column_in_batches(table, column, default) + end + # We want to rescue _all_ exceptions here, even those that don't inherit + # from StandardError. + rescue Exception => error # rubocop: disable all + remove_column(table, column) + + raise error + end + + change_column_null(table, column, false) unless allow_null + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index dccb717e95d..87a9b1e23ac 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,6 +1,11 @@ module Gitlab module Diff class InlineDiffMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + } + attr_accessor :raw_line, :rich_line def initialize(raw_line, rich_line = raw_line) @@ -8,7 +13,7 @@ module Gitlab @rich_line = ERB::Util.html_escape(rich_line) end - def mark(line_inline_diffs) + def mark(line_inline_diffs, mode: nil, markdown: false) return rich_line unless line_inline_diffs marker_ranges = [] @@ -20,13 +25,22 @@ module Gitlab end offset = 0 - # Mark each range - marker_ranges.each_with_index do |range, i| - class_names = ["idiff"] - class_names << "left" if i == 0 - class_names << "right" if i == marker_ranges.length - 1 - offset = insert_around_range(rich_line, range, "<span class='#{class_names.join(" ")}'>", "</span>", offset) + # Mark each range + marker_ranges.each_with_index do |range, index| + before_content = + if markdown + "{#{MARKDOWN_SYMBOLS[mode]}" + else + "<span class='#{html_class_names(marker_ranges, mode, index)}'>" + end + after_content = + if markdown + "#{MARKDOWN_SYMBOLS[mode]}}" + else + "</span>" + end + offset = insert_around_range(rich_line, range, before_content, after_content, offset) end rich_line.html_safe @@ -34,6 +48,14 @@ module Gitlab private + def html_class_names(marker_ranges, mode, index) + class_names = ["idiff"] + class_names << "left" if index == 0 + class_names << "right" if index == marker_ranges.length - 1 + class_names << mode if mode + class_names.join(" ") + end + # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index d0815fc7eea..522dd2b9428 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,16 +17,16 @@ module Gitlab Enumerator.new do |yielder| @lines.each do |line| next if filename?(line) - - full_line = line.gsub(/\n/, '') - + + full_line = line.delete("\n") + if line.match(/^@@ -/) type = "match" - + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 #top of file + + next if line_old <= 1 && line_new <= 1 # top of file yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next @@ -39,8 +39,8 @@ module Gitlab yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 end - - + + case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 8f9be6cd9a3..e2fee6b9f3e 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -2,22 +2,21 @@ module Gitlab module Email module Message class RepositoryPush - attr_accessor :recipient attr_reader :author_id, :ref, :action include Gitlab::Routing.url_helpers + include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author delegate :username, to: :author, prefix: :author - def initialize(notify, project_id, recipient, opts = {}) + def initialize(notify, project_id, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless opts[:author_id] && opts[:ref] && opts[:action] @notify = notify @project_id = project_id - @recipient = recipient @opts = opts.dup @author_id = @opts.delete(:author_id) @@ -38,7 +37,7 @@ module Gitlab end def diffs - @diffs ||= (compare.diffs if compare) + @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) end def diffs_count @@ -49,6 +48,10 @@ module Gitlab @opts[:compare] end + def diff_refs + @opts[:diff_refs] + end + def compare_timeout diffs.overflow? if diffs end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 6ed36b51f12..3411eb1d9ce 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -65,7 +65,7 @@ module Gitlab (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } + break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } # Headers on the same line break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 3840765db87..1918d5b208d 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.safe_name, path: repo.path, @@ -21,12 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::INTERNAL, import_type: 'fogbugz', import_source: repo.name, - import_url: Project::UNKNOWN_IMPORT_URL + import_url: Project::UNKNOWN_IMPORT_URL, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session } } ).execute - - project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session }) - - project end end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 202263c6742..72992baffd4 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -9,6 +9,10 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new end + def create! + self.klass.create!(self.attributes) + end + private def gl_user_id(github_id) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb new file mode 100644 index 00000000000..a15fc84b418 --- /dev/null +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -0,0 +1,29 @@ +module Gitlab + module GithubImport + class BranchFormatter < BaseFormatter + delegate :repo, :sha, :ref, to: :raw_data + + def exists? + project.repository.branch_exists?(ref) + end + + def name + @name ||= exists? ? ref : "#{ref}-#{short_id}" + end + + def valid? + repo.present? + end + + def valid? + repo.present? + end + + private + + def short_id + sha.to_s[0..7] + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 67988ea3460..d325eca6d99 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,6 +1,9 @@ module Gitlab module GithubImport class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :client, :api def initialize(access_token) @@ -11,7 +14,7 @@ module Gitlab ) if access_token - ::Octokit.auto_paginate = true + ::Octokit.auto_paginate = false @api = ::Octokit::Client.new( access_token: access_token, @@ -36,7 +39,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - api.send(method, *args, &block) + request { api.send(method, *args, &block) } else super(method, *args, &block) end @@ -55,6 +58,34 @@ module Gitlab def github_options config["args"]["client_options"].deep_symbolize_keys end + + def rate_limit + api.rate_limit! + end + + def rate_limit_exceed? + rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = yield + + last_response = api.last_response + + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + data.concat(last_response.data) if last_response.data.is_a?(Array) + end + + data + end end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 7d679eaec6a..2c1b94ef2cd 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -8,6 +8,7 @@ module Gitlab commit_id: raw_data.commit_id, line_code: line_code, author_id: author_id, + type: type, created_at: raw_data.created_at, updated_at: raw_data.updated_at } @@ -53,6 +54,10 @@ module Gitlab def note formatter.author_line(author) + body end + + def type + 'LegacyDiffNote' if on_diff? + end end end end diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb new file mode 100644 index 00000000000..db1fabaa18a --- /dev/null +++ b/lib/gitlab/github_import/hook_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class HookFormatter + EVENTS = %w[* create delete pull_request push].freeze + + attr_reader :raw + + delegate :id, :name, :active, to: :raw + + def initialize(raw) + @raw = raw + end + + def config + raw.config.attrs + end + + def valid? + (EVENTS & raw.events).any? && active + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 0f9e3ee14ee..e5cf66a0371 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,12 +3,15 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :project, :client + attr_reader :client, :project, :repo, :repo_url def initialize(project) - @project = project - if import_data_credentials - @client = Client.new(import_data_credentials[:user]) + @project = project + @repo = project.import_source + @repo_url = project.import_url + + if credentials + @client = Client.new(credentials[:user]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -22,14 +25,13 @@ module Gitlab private - def import_data_credentials - @import_data_credentials ||= project.import_data.credentials if project.import_data + def credentials + @credentials ||= project.import_data.credentials if project.import_data end def import_labels - client.labels(project.import_source).each do |raw_data| - Label.create!(LabelFormatter.new(project, raw_data).attributes) - end + labels = client.labels(repo, per_page: 100) + labels.each { |raw| LabelFormatter.new(project, raw).create! } true rescue ActiveRecord::RecordInvalid => e @@ -37,9 +39,8 @@ module Gitlab end def import_milestones - client.list_milestones(project.import_source, state: :all).each do |raw_data| - Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) - end + milestones = client.milestones(repo, state: :all, per_page: 100) + milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } true rescue ActiveRecord::RecordInvalid => e @@ -47,18 +48,15 @@ module Gitlab end def import_issues - client.list_issues(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| - gh_issue = IssueFormatter.new(project, raw_data) + issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - if gh_issue.valid? - issue = Issue.create!(gh_issue.attributes) - apply_labels(gh_issue.number, issue) + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw) - if gh_issue.has_comments? - import_comments(gh_issue.number, issue) - end + if gh_issue.valid? + issue = gh_issue.create! + apply_labels(issue) + import_comments(issue) if gh_issue.has_comments? end end @@ -68,29 +66,64 @@ module Gitlab end def import_pull_requests - client.pull_requests(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| - pull_request = PullRequestFormatter.new(project, raw_data) - - if pull_request.valid? - merge_request = MergeRequest.new(pull_request.attributes) - - if merge_request.save - apply_labels(pull_request.number, merge_request) - import_comments(pull_request.number, merge_request) - import_comments_on_diff(pull_request.number, merge_request) - end - end + hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + disable_webhooks(hooks) + + pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) + pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) + + source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } + target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } + branches_removed = source_branches_removed | target_branches_removed + + restore_branches(branches_removed) + + pull_requests.each do |pull_request| + merge_request = pull_request.create! + apply_labels(merge_request) + import_comments(merge_request) + import_comments_on_diff(merge_request) end true rescue ActiveRecord::RecordInvalid => e raise Projects::ImportService::Error, e.message + ensure + clean_up_restored_branches(branches_removed) + clean_up_disabled_webhooks(hooks) + end + + def disable_webhooks(hooks) + update_webhooks(hooks, active: false) + end + + def clean_up_disabled_webhooks(hooks) + update_webhooks(hooks, active: true) + end + + def update_webhooks(hooks, options) + hooks.each do |hook| + client.edit_hook(repo, hook.id, hook.name, hook.config, options) + end + end + + def restore_branches(branches) + branches.each do |name, sha| + client.create_ref(repo, "refs/heads/#{name}", sha) + end + + project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') + end + + def clean_up_restored_branches(branches) + branches.each do |name, _| + client.delete_ref(repo, "heads/#{name}") + project.repository.rm_branch(project.creator, name) + end end - def apply_labels(number, issuable) - issue = client.issue(project.import_source, number) + def apply_labels(issuable) + issue = client.issue(repo, issuable.iid) if issue.labels.count > 0 label_ids = issue.labels.map do |raw| @@ -101,20 +134,20 @@ module Gitlab end end - def import_comments(issue_number, noteable) - comments = client.issue_comments(project.import_source, issue_number) - create_comments(comments, noteable) + def import_comments(issuable) + comments = client.issue_comments(repo, issuable.iid, per_page: 100) + create_comments(issuable, comments) end - def import_comments_on_diff(pull_request_number, merge_request) - comments = client.pull_request_comments(project.import_source, pull_request_number) - create_comments(comments, merge_request) + def import_comments_on_diff(merge_request) + comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) + create_comments(merge_request, comments) end - def create_comments(comments, noteable) - comments.each do |raw_data| - comment = CommentFormatter.new(project, raw_data) - noteable.notes.create!(comment.attributes) + def create_comments(issuable, comments) + comments.each do |raw| + comment = CommentFormatter.new(project, raw) + issuable.notes.create!(comment.attributes) end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index c8173913b4e..835ec858b35 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -20,6 +20,10 @@ module Gitlab raw_data.comments > 0 end + def klass + Issue + end + def number raw_data.number end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index c2b9d40b511..9f18244e7d7 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -9,6 +9,10 @@ module Gitlab } end + def klass + Label + end + private def color diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index e91a7e328cf..53d4b3102d1 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -14,6 +14,10 @@ module Gitlab } end + def klass + Milestone + end + private def number diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index d21b942ad4b..498b00cb658 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,15 +1,20 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter + delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true + delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true + def attributes { iid: number, title: raw_data.title, description: description, - source_project: source_project, - source_branch: source_branch.name, - target_project: target_project, - target_branch: target_branch.name, + source_project: source_branch_project, + source_branch: source_branch_name, + head_source_sha: source_branch_sha, + target_project: target_branch_project, + target_branch: target_branch_name, + base_target_sha: target_branch_sha, state: state, milestone: milestone, author_id: author_id, @@ -19,12 +24,24 @@ module Gitlab } end + def klass + MergeRequest + end + def number raw_data.number end def valid? - !cross_project? && source_branch.present? && target_branch.present? + source_branch.valid? && target_branch.valid? && !cross_project? + end + + def source_branch + @source_branch ||= BranchFormatter.new(project, raw_data.head) + end + + def target_branch + @target_branch ||= BranchFormatter.new(project, raw_data.base) end private @@ -52,7 +69,7 @@ module Gitlab end def cross_project? - source_repo.present? && target_repo.present? && source_repo.id != target_repo.id + source_branch_repo.id != target_branch_repo.id end def description @@ -65,35 +82,10 @@ module Gitlab end end - def source_project - project - end - - def source_repo - raw_data.head.repo - end - - def source_branch - source_project.repository.find_branch(raw_data.head.ref) - end - - def target_project - project - end - - def target_repo - raw_data.base.repo - end - - def target_branch - target_project.repository.find_branch(raw_data.base.ref) - end - def state - @state ||= case true - when raw_data.state == 'closed' && raw_data.merged_at.present? + @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? 'merged' - when raw_data.state == 'closed' + elsif raw_data.state == 'closed' 'closed' else 'opened' diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb new file mode 100644 index 00000000000..f46b43b61a4 --- /dev/null +++ b/lib/gitlab/gitignore.rb @@ -0,0 +1,56 @@ +module Gitlab + class Gitignore + FILTER_REGEX = /\.gitignore\z/.freeze + + def initialize(path) + @path = path + end + + def name + File.basename(@path, '.gitignore') + end + + def content + File.read(@path) + end + + class << self + def all + languages_frameworks + global + end + + def find(key) + file_name = "#{key}.gitignore" + + directory = select_directory(file_name) + directory ? new(File.join(directory, file_name)) : nil + end + + def global + files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } + end + + def languages_frameworks + files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } + end + + private + + def select_directory(file_name) + [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } + end + + def global_dir + File.join(gitignore_dir, 'Global') + end + + def gitignore_dir + Rails.root.join('vendor/gitignore') + end + + def files_for_folder(dir) + Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 96717b42bae..3f76ec97977 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,9 +5,9 @@ module Gitlab def initialize(project) @project = project - credentials = import_data - if credentials && credentials[:password] - @client = Client.new(credentials[:password]) + import_data = project.import_data + if import_data && import_data.credentials && import_data.credentials[:password] + @client = Client.new(import_data.credentials[:password]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -17,7 +17,7 @@ module Gitlab def execute project_identifier = CGI.escape(project.import_source) - #Issues && Comments + # Issues && Comments issues = client.issues(project_identifier) issues.each do |issue| diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ab900b641c4..f751a3a12fd 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,6 +8,7 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_shortcuts_path gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.award_menu_url = emojis_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 0abb7a64c17..326cfcaa8af 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -21,12 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::PUBLIC, import_type: "google_code", import_source: repo.name, - import_url: repo.import_url + import_url: repo.import_url, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } ).execute - - project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }) - - project end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 9c04a3d6995..624c1766024 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -20,6 +20,10 @@ module Gitlab "project.bundle" end + def config_file + 'lib/gitlab/import_export/import_export.yml' + end + def version_filename 'VERSION' end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index eef4d92beee..3796fc8cd02 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -5,7 +5,9 @@ project_tree: :author - :labels - :milestones - - :snippets + - snippets: + - notes: + :author - :releases - :events - project_members: @@ -14,7 +16,9 @@ project_tree: - notes: :author - :merge_request_diff - - ci_commits: + - pipelines: + - notes: + :author - :statuses - :variables - :triggers diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index d42005b2bfd..25253330745 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -4,17 +4,19 @@ module Gitlab attr_reader :tree - def initialize(config: 'lib/gitlab/import_export/import_export.yml', shared:) + def initialize(shared:) @shared = shared - config_hash = YAML.load_file(config).deep_symbolize_keys + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys @tree = config_hash[:project_tree] - @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes], methods: config_hash[:methods]) end + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. def project_tree - @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) rescue => e @shared.error(e) false @@ -22,23 +24,30 @@ module Gitlab private + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file def build_hash(model_list) model_list.map do |model_objects| if model_objects.is_a?(Hash) build_json_config_hash(model_objects) else - @attributes_parser.find(model_objects) + @attributes_finder.find(model_objects) end end end + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} def build_json_config_hash(model_object_hash) @json_config_hash = {} model_object_hash.values.flatten.each do |model_object| current_key = model_object_hash.keys.first - @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } handle_model_object(current_key, model_object) process_sub_model(current_key, model_object) if model_object.is_a?(Hash) @@ -46,6 +55,11 @@ module Gitlab @json_config_hash end + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash def process_sub_model(current_key, model_object) sub_model_json = build_json_config_hash(model_object).dup @json_config_hash.slice!(current_key) @@ -57,6 +71,9 @@ module Gitlab end end + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash def handle_model_object(current_key, model_object) if @json_config_hash[current_key] add_model_value(current_key, model_object) @@ -65,21 +82,33 @@ module Gitlab end end + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash def create_model_value(current_key, value) parsed_hash = { include: value } - @attributes_parser.parse(value) do |hash| + @attributes_finder.parse(value) do |hash| parsed_hash = { include: hash_or_merge(value, hash) } end @json_config_hash[current_key] = parsed_hash end + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash def add_model_value(current_key, value) - @attributes_parser.parse(value) { |hash| value = { value => hash } } + @attributes_finder.parse(value) { |hash| value = { value => hash } } old_values = @json_config_hash[current_key][:include] @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten end + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ def hash_or_merge(value, hash) value.is_a?(Hash) ? value.merge(hash) : { value => hash } end diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 79794a4c34f..95d4fb17ead 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -16,8 +16,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) - if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo, restore_uploads].all? - Todo.create(attributes_for_todo) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.project else project_tree.project.destroy if project_tree.project @@ -27,12 +26,8 @@ module Gitlab private - def restore_version - Gitlab::ImportExport::VersionRestorer.restore(shared: @shared) - end - - def restore_project_tree - project_tree.restore + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) end def project_tree @@ -41,20 +36,21 @@ module Gitlab namespace_id: @namespace.id) end - def restore_repo + def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: @shared, - project: project_tree.project).restore + project: project_tree.project) end - def restore_wiki_repo + def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.project)).restore + project: ProjectWiki.new(project_tree.project), + wiki: true) end - def restore_uploads - Gitlab::ImportExport::UploadsRestorer.restore(project: project_tree.project, shared: @shared) + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) end def path_with_namespace(project_path) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index cd1e174f180..13fe68d5408 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :map, :note_member_list + attr_reader :note_member_list def initialize(exported_members:, user:, project:) @exported_members = exported_members @@ -12,50 +12,55 @@ module Gitlab # This needs to run first, as second call would be from generate_map # which means project members already exist. - default_project_member - - @project_member_map = Hash.new do |_, key| - @note_member_list << key - default_project_member - end + ensure_default_member! + end - @map = generate_map + def map + @map ||= generate_map end - def default_project_member - @default_project_member ||= - begin - default_member = ProjectMember.new(default_project_member_hash) - default_member.save! - default_member.user.id - end + def default_user_id + @user.id end private + def generate_map - @exported_members.each do |member| - existing_user = User.where(find_project_user_query(member)).first - assign_member(existing_user, member) if existing_user + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member).persisted? + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def missing_keys_tracking_hash + Hash.new do |_, key| + @note_member_list << key + @user.id end - @project_member_map end - def assign_member(existing_user, member) - old_user_id = member['user']['id'] + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) member['user'] = existing_user - project_member = ProjectMember.new(member_hash(member)) - @project_member_map[old_user_id] = project_member.user.id if project_member.save + + ProjectMember.create(member_hash(member)) end def member_hash(member) member.except('id').merge(source_id: @project.id, importing: true) end - def default_project_member_hash - { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true } - end - def find_project_user_query(member) user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 911ba06e748..dc1e477a82f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -32,14 +32,20 @@ module Gitlab project: project) end - def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations saved = [] - relation_list.each do |relation| - next if !relation.is_a?(Hash) && tree_hash[relation.to_s].blank? - create_sub_relations(relation, tree_hash) if relation.is_a?(Hash) + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash = create_relation(relation_key, tree_hash[relation_key.to_s]) + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) saved << project.update_attribute(relation_key, relation_hash) end saved.all? @@ -73,31 +79,20 @@ module Gitlab relation_hash = relation_item[sub_relation.to_s] end - process_sub_relation(relation_hash, relation_item, sub_relation) unless relation_hash.blank? + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end end end - def process_sub_relation(relation_hash, relation_item, sub_relation) - if relation_hash.is_a?(Array) - sub_relation_object = create_relation(sub_relation, relation_hash) - else - sub_relation_object = relation_from_factory(sub_relation, relation_hash) - end - relation_item[sub_relation.to_s] = sub_relation_object - end - def create_relation(relation, relation_hash_list) - [relation_hash_list].flatten.map do |relation_hash| - relation_from_factory(relation, relation_hash) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => project.id), + members_mapper: members_mapper, + user_admin: @user.is_admin?) end - end - def relation_from_factory(relation, relation_hash) - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => project.id), - members_mapper: members_mapper, - user_admin: @user.is_admin?) + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0fdf208cfa0..486e09c9a39 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,56 +1,65 @@ module Gitlab module ImportExport - module RelationFactory - extend self + class RelationFactory OVERRIDES = { snippets: :project_snippets, - ci_commits: 'Ci::Commit', + pipelines: 'Ci::Pipeline', statuses: 'commit_status', variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', hooks: 'ProjectHook' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze - def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) - relation_sym = parse_relation_sym(relation_sym) - klass = parse_relation(relation_hash, relation_sym) + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user_admin:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user_admin = user_admin + end - update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes - update_user_references(relation_hash, members_mapper.map) - update_project_references(relation_hash, klass) - reset_tokens(relation_hash) if relation_sym == 'Ci::Trigger' + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' - generate_imported_object(klass, relation_hash, relation_sym) + generate_imported_object end private - def update_user_references(relation_hash, members_map) + def update_user_references USER_REFERENCES.each do |reference| - if relation_hash[reference] - relation_hash[reference] = members_map[relation_hash[reference]] + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end end - def update_missing_author(relation_hash, members_map, user_admin) - old_author_id = relation_hash['author_id'] + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] # Users with admin access can map users - if user_admin - relation_hash['author_id'] = members_map.map[old_author_id] - else - relation_hash['author_id'] = members_map.default_project_member - end - - author = relation_hash.delete('author') + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - return unless user_admin && members_map.note_member_list.include?(old_author_id) + author = @relation_hash.delete('author') - relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? - relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], author['name'])) + if admin_user? && @members_mapper.note_member_list.include?(old_author_id) + update_note_for_missing_author(author['name']) + end end def missing_author_note(updated_at, author_name) @@ -58,62 +67,59 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end - def generate_imported_object(klass, relation_hash, relation_sym) - if relation_sym == 'commit_status' # call #trace= method after assigning the other attributes - trace = relation_hash.delete('trace') - imported_object(klass, relation_hash) do |imported_object| - imported_object.trace = trace - imported_object.commit_id = nil + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil end else - imported_object(klass, relation_hash) + imported_object end end - def update_project_references(relation_hash, klass) - project_id = relation_hash.delete('project_id') - - if relation_hash['source_project_id'] && relation_hash['target_project_id'] - # If source and target are the same, populate them with the new project ID. - if relation_hash['target_project_id'] == relation_hash['source_project_id'] - relation_hash['source_project_id'] = project_id - else - relation_hash['source_project_id'] = -1 - end - end - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + def update_project_references + project_id = @relation_hash.delete('project_id') # project_id may not be part of the export, but we always need to populate it if required. - relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') - relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end end - def reset_tokens(relation_hash) + def reset_ci_tokens return unless Gitlab::ImportExport.reset_tokens? - # If we import/export a project to the same instance, tokens will have to be reseated. - relation_hash['token'] = nil + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil end - def relation_class(relation_sym) - relation_sym.to_s.classify.constantize + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize end - def parse_relation_sym(relation_sym) - OVERRIDES[relation_sym] || relation_sym - end - - def imported_object(klass, relation_hash) - imported_object = klass.new(relation_hash) + def imported_object + imported_object = relation_class.new(@relation_hash) yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end - def parse_relation(relation_hash, relation_sym) - klass = relation_class(relation_sym) - relation_hash.delete('id') - klass + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user_admin end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 36094b95aa4..d6dcf5dc116 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,16 +3,16 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) + def initialize(project:, shared:, path_to_bundle:, wiki: false) @project = project @path_to_bundle = path_to_bundle @shared = shared + @wiki = wiki end def restore - return true unless File.exists?(@path_to_bundle) + return false unless File.exists?(@path_to_bundle) || wiki? - FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) @@ -30,6 +30,10 @@ module Gitlab def path_to_repo @project.repository.path_to_repo end + + def wiki? + @wiki + end end end end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_saver.rb index 377dfcfe2c9..07d71c78f33 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -1,6 +1,6 @@ module Gitlab module ImportExport - class RepoBundler + class RepoSaver include Gitlab::ImportExport::CommandLineUtil attr_reader :full_path @@ -10,7 +10,7 @@ module Gitlab @shared = shared end - def bundle + def save return false if @project.empty_repo? @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 01ac332981b..6aff05b886a 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -10,7 +10,7 @@ module Gitlab end def export_path - @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) end def error(error) diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index fdcbc48eb1b..df19354b76e 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -1,12 +1,7 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver - - class << self - alias_method :restore, :save - end - - def save + def restore return true unless File.directory?(uploads_export_path) copy_files(uploads_export_path, uploads_path) diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 93bc626b363..7292e9d9712 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -2,10 +2,6 @@ module Gitlab module ImportExport class UploadsSaver - def self.save(*args) - new(*args).save - end - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_checker.rb index 892352f5adf..4f467760862 100644 --- a/lib/gitlab/import_export/version_restorer.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -1,16 +1,16 @@ module Gitlab module ImportExport - class VersionRestorer + class VersionChecker def self.restore(*args) - new(*args).restore + new(*args).check end def initialize(shared:) @shared = shared end - def restore + def check! version = File.open(version_file, &:readline) verify_version!(version) rescue => e diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 59e317e6573..fe02a2a6d76 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -2,10 +2,6 @@ module Gitlab module ImportExport class VersionSaver - def self.save(*args) - new(*args).save - end - def initialize(shared:) @shared = shared end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 015f54d17af..1eedae39f8a 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport - class WikiRepoBundler < RepoBundler - def bundle + class WikiRepoSaver < RepoSaver + def save @wiki = ProjectWiki.new(@project) return true unless wiki_repository_exists? # it's okay to have no Wiki bundle_to_disk(File.join(@shared.export_path, project_filename)) @@ -26,7 +26,7 @@ module Gitlab end def wiki_repository_exists? - File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? end end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index baf52ff750d..8684b4636ea 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -17,9 +17,9 @@ module Gitlab file.rewind cmd = [] - cmd.push *%W(ssh-keygen) - cmd.push *%W(-E md5) if explicit_fingerprint_algorithm? - cmd.push *%W(-lf #{file.path}) + cmd.push('ssh-keygen') + cmd.push('-E', 'md5') if explicit_fingerprint_algorithm? + cmd.push('-lf', file.path) cmd_output, cmd_status = popen(cmd, '/tmp') end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb new file mode 100644 index 00000000000..2a659ae4c74 --- /dev/null +++ b/lib/gitlab/lazy.rb @@ -0,0 +1,34 @@ +module Gitlab + # A class that can be wrapped around an expensive method call so it's only + # executed when actually needed. + # + # Usage: + # + # object = Gitlab::Lazy.new { some_expensive_work_here } + # + # object['foo'] + # object.bar + class Lazy < BasicObject + def initialize(&block) + @block = block + end + + def method_missing(name, *args, &block) + __evaluate__ + + @result.__send__(name, *args, &block) + end + + def respond_to_missing?(name, include_private = false) + __evaluate__ + + @result.respond_to?(name, include_private) || super + end + + private + + def __evaluate__ + @result = @block.call unless defined?(@result) + end + end +end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index aff7ccb157f..f9bb5775323 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -93,6 +93,7 @@ module Gitlab end protected + def base_config Gitlab.config.ldap end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index a5f767b134d..dda371e6554 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -40,7 +40,7 @@ module Gitlab # Returns boolean def plain?(filename) filename.downcase.end_with?('.txt') || - filename.downcase == 'readme' + filename.casecmp('readme').zero? end def previewable?(filename) diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 708ef79f304..0f115893a15 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -154,8 +154,6 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold - trans.increment(:method_duration, duration) - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 49e5f86e6e6..8e345e8ae4a 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -6,26 +6,28 @@ module Gitlab attach_to :active_support def cache_read(event) - increment(:cache_read_duration, event.duration) + increment(:cache_read, event.duration) end def cache_write(event) - increment(:cache_write_duration, event.duration) + increment(:cache_write, event.duration) end def cache_delete(event) - increment(:cache_delete_duration, event.duration) + increment(:cache_delete, event.duration) end def cache_exist?(event) - increment(:cache_exists_duration, event.duration) + increment(:cache_exists, event.duration) end def increment(key, duration) return unless current_transaction current_transaction.increment(:cache_duration, duration) - current_transaction.increment(key, duration) + current_transaction.increment(:cache_count, 1) + current_transaction.increment("#{key}_duration".to_sym, duration) + current_transaction.increment("#{key}_count".to_sym, 1) end private diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 50b0dd32380..5764ab15652 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -39,7 +39,7 @@ module Gitlab request_url = URI.join(base_url, project_path) domain_path = strip_url(request_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n"; + "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n" end def strip_url(url) diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb new file mode 100644 index 00000000000..56608b1b276 --- /dev/null +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to measure the latency between +# gitlab-workhorse forwarding a request to the Rails application and the +# time this middleware is reached. + +module Gitlab + module Middleware + class RailsQueueDuration + def initialize(app) + @app = app + end + + def call(env) + trans = Gitlab::Metrics.current_transaction + proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + if trans && proxy_start + # Time in milliseconds since gitlab-workhorse started the request + trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 356e96fcbab..78f3ecb4cb4 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -69,13 +69,20 @@ module Gitlab return unless ldap_person # If a corresponding person exists with same uid in a LDAP server, - # set up a Gitlab user with dual LDAP and Omniauth identities. - if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) - # Case when a LDAP user already exists in Gitlab. Add the Omniauth identity to existing account. + # check if the user already has a GitLab account. + user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) + if user + # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. + log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) else - # No account in Gitlab yet: create it and add the LDAP identity - user = build_new_user + log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." + user = find_by_uid_and_provider + if user.nil? + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + user = build_new_user + end + log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 71c5b6801fb..183bd10d6a3 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -74,7 +74,7 @@ module Gitlab end def notes - project.notes.user.search(query).order('updated_at DESC') + project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') end def commits diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 5c352c96de5..40766f35f77 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -25,7 +25,7 @@ module Gitlab end @pool.with { |redis| yield redis } end - + def self.redis_store_options url = new.url redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) @@ -40,10 +40,10 @@ module Gitlab def initialize(rails_env=nil) rails_env ||= Rails.env config_file = File.expand_path('../../../config/resque.yml', __FILE__) - + @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] + if File.exist?(config_file) + @url = YAML.load_file(config_file)[rails_env] end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 13c4d64c99b..11c0b01f0dc 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,10 +4,9 @@ module Gitlab REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil, author = nil) + def initialize(project, current_user = nil) @project = project @current_user = current_user - @author = author @references = {} @@ -18,17 +17,21 @@ module Gitlab super(text, context.merge(project: project)) end + def references(type) + super(type, project, current_user) + end + REFERABLES.each do |type| define_method("#{type}s") do - @references[type] ||= references(type, reference_context) + @references[type] ||= references(type) end end def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue, reference_context) + @references[:external_issue] ||= references(:external_issue) else - @references[:issue] ||= references(:issue, reference_context) + @references[:issue] ||= references(:issue) end end @@ -46,11 +49,5 @@ module Gitlab @pattern = Regexp.union(patterns.compact) end - - private - - def reference_context - { project: project, current_user: current_user, author: author } - end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,9 @@ module Gitlab (?<![\/.]) (?# rule #6-7) }x.freeze end + + def container_registry_reference_regex + git_reference_regex + end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index dba4bbfc899..8943022612c 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -12,12 +12,12 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider - if auto_link_ldap_user? @user ||= find_or_create_ldap_user end + @user ||= find_by_uid_and_provider + if auto_link_saml_user? @user ||= find_by_email end diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index b98589dff89..8304b9a482c 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -1,5 +1,3 @@ -require_relative "svg/whitelist" - module Gitlab module Sanitizers module SVG @@ -12,25 +10,47 @@ module Gitlab DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u def scrub(node) - unless ALLOWED_ELEMENTS.include?(node.name) + unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) node.unlink - else - node.attributes.each do |attr_name, attr| - valid_attributes = ALLOWED_ATTRIBUTES[node.name] - - unless valid_attributes && valid_attributes.include?(attr_name) - if ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name) && - attr_name.start_with?('data-') - # Arbitrary data attributes are allowed. Verify that the attribute - # is a valid data attribute. - attr.unlink unless attr_name =~ DATA_ATTR_PATTERN - else - attr.unlink - end + return + end + + valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name] + return unless valid_attributes + + node.attribute_nodes.each do |attr| + attr_name = attribute_name_with_namespace(attr) + + if valid_attributes.include?(attr_name) + attr.unlink if unsafe_href?(attr) + else + # Arbitrary data attributes are allowed. + unless allows_data_attribute?(node) && data_attribute?(attr) + attr.unlink end end end end + + def attribute_name_with_namespace(attr) + if attr.namespace + "#{attr.namespace.prefix}:#{attr.name}" + else + attr.name + end + end + + def allows_data_attribute?(node) + Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name) + end + + def unsafe_href?(attr) + attribute_name_with_namespace(attr) == 'xlink:href' && !attr.value.start_with?('#') + end + + def data_attribute?(attr) + attr.name.start_with?('data-') && attr.name =~ DATA_ATTR_PATTERN && attr.namespace.nil? + end end end end diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb index 917e795b29e..7b6b70d8dbc 100644 --- a/lib/gitlab/sanitizers/svg/whitelist.rb +++ b/lib/gitlab/sanitizers/svg/whitelist.rb @@ -4,7 +4,8 @@ module Gitlab module Sanitizers module SVG - ALLOWED_ELEMENTS = %w[ + class Whitelist + ALLOWED_ELEMENTS = %w[ a altGlyph altGlyphDef altGlyphItem animate animateColor animateMotion animateTransform circle clipPath color-profile cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer @@ -18,90 +19,91 @@ module Gitlab script set stop style svg switch symbol text textPath title tref tspan use view vkern].freeze - ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze + ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze - ALLOWED_ATTRIBUTES = { - 'a' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'altGlyphDef' => %w[id xml:base xml:lang xml:space], - 'altGlyphItem' => %w[id xml:base xml:lang xml:space], - 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'desc' => %w[class id style xml:base xml:lang xml:space], - 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], - 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], - 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feMergeNode' => %w[id xml:base xml:lang xml:space], - 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], - 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], - 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], - 'font-face-format' => %w[id string xml:base xml:lang xml:space], - 'font-face-name' => %w[id name xml:base xml:lang xml:space], - 'font-face-src' => %w[id xml:base xml:lang xml:space], - 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], - 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], - 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], - 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'metadata' => %w[id xml:base xml:lang xml:space], - 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'style' => %w[id media title type xml:base xml:lang xml:space], - 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], - 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'title' => %w[class id style xml:base xml:lang xml:space], - 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], - 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], - 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] - }.freeze + ALLOWED_ATTRIBUTES = { + 'a' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'altGlyphDef' => %w[id xml:base xml:lang xml:space], + 'altGlyphItem' => %w[id xml:base xml:lang xml:space], + 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'desc' => %w[class id style xml:base xml:lang xml:space], + 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], + 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], + 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feMergeNode' => %w[id xml:base xml:lang xml:space], + 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], + 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], + 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], + 'font-face-format' => %w[id string xml:base xml:lang xml:space], + 'font-face-name' => %w[id name xml:base xml:lang xml:space], + 'font-face-src' => %w[id xml:base xml:lang xml:space], + 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], + 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], + 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], + 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'metadata' => %w[id xml:base xml:lang xml:space], + 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'style' => %w[id media title type xml:base xml:lang xml:space], + 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], + 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'title' => %w[class id style xml:base xml:lang xml:space], + 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], + 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], + 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] + }.freeze + end end end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 2ef0e982256..7cf506ebe64 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -5,7 +5,7 @@ module Gitlab SeedFu.quiet = true yield SeedFu.quiet = false - puts "\nOK".green + puts "\nOK".color(:green) end def self.by_user(user) diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 2bbbd3074e8..fe65c246101 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -62,7 +62,7 @@ module Gitlab end def wiki_page_url - "#{Gitlab.config.gitlab.url}#{object.wiki.wiki_base_path}/#{object.slug}" + namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/url_sanitizer.rb index d23b013c1f5..7d02fe3c971 100644 --- a/lib/gitlab/import_url.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,13 @@ module Gitlab - class ImportUrl + class UrlSanitizer + def self.sanitize(content) + regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) + + content.gsub(regexp) { |url| new(url).masked_url } + end + def initialize(url, credentials: nil) - @url = URI.parse(URI.encode(url)) + @url = Addressable::URI.parse(url) @credentials = credentials end @@ -9,6 +15,13 @@ module Gitlab @sanitized_url ||= safe_url.to_s end + def masked_url + url = @url.dup + url.password = "*****" unless url.password.nil? + url.user = "*****" unless url.user.nil? + url.to_s + end + def credentials @credentials ||= { user: @url.user, password: @url.password } end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index a1ee1cba216..9462f3368e6 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -32,6 +32,13 @@ module Gitlab } end + def highest_allowed_level + restricted_levels = current_application_settings.restricted_visibility_levels + + allowed_levels = self.values - restricted_levels + allowed_levels.max || PRIVATE + end + def allowed_for?(user, level) user.is_admin? || allowed_level?(level.to_i) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c3ddd4c2680..388f84dbe0e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -6,6 +6,13 @@ module Gitlab SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' class << self + def git_http_ok(repository, user) + { + 'GL_ID' => Gitlab::ShellEnv.gl_id(user), + 'RepoPath' => repository.path_to_repo, + } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -14,24 +21,39 @@ module Gitlab [ SEND_DATA_HEADER, - "git-blob:#{encode(params)}", + "git-blob:#{encode(params)}" ] end - def send_git_archive(project, ref, format) + def send_git_archive(repository, ref:, format:) format ||= 'tar.gz' format.downcase! - params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? [ SEND_DATA_HEADER, - "git-archive:#{encode(params)}", + "git-archive:#{encode(params)}" ] end - + + def send_git_diff(repository, diff_refs) + from, to = diff_refs + + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => from.sha, + 'ShaTo' => to.sha + } + + [ + SEND_DATA_HEADER, + "git-diff:#{encode(params)}" + ] + end + protected - + def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb new file mode 100644 index 00000000000..d6d6af7089c --- /dev/null +++ b/lib/json_web_token/rsa_token.rb @@ -0,0 +1,42 @@ +module JSONWebToken + class RSAToken < Token + attr_reader :key_file + + def initialize(key_file) + super() + @key_file = key_file + end + + def encoded + headers = { + kid: kid + } + JWT.encode(payload, key, 'RS256', headers) + end + + private + + def key_data + @key_data ||= File.read(key_file) + end + + def key + @key ||= OpenSSL::PKey::RSA.new(key_data) + end + + def public_key + key.public_key + end + + def kid + # calculate sha256 from DER encoded ASN1 + kid = Digest::SHA256.digest(public_key.to_der) + + # we encode only 30 bytes with base32 + kid = Base32.encode(kid[0..29]) + + # insert colon every 4 characters + kid.scan(/.{4}/).join(':') + end + end +end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb new file mode 100644 index 00000000000..5b67715b0b2 --- /dev/null +++ b/lib/json_web_token/token.rb @@ -0,0 +1,46 @@ +module JSONWebToken + class Token + attr_accessor :issuer, :subject, :audience, :id + attr_accessor :issued_at, :not_before, :expire_time + + def initialize + @id = SecureRandom.uuid + @issued_at = Time.now + # we give a few seconds for time shift + @not_before = issued_at - 5.seconds + # default 60 seconds should be more than enough for this authentication token + @expire_time = issued_at + 1.minute + @custom_payload = {} + end + + def [](key) + @custom_payload[key] + end + + def []=(key, value) + @custom_payload[key] = value + end + + def encoded + raise NotImplementedError + end + + def payload + @custom_payload.merge(default_payload) + end + + private + + def default_payload + { + jti: id, + aud: audience, + sub: subject, + iss: issuer, + iat: issued_at.to_i, + nbf: not_before.to_i, + exp: expire_time.to_i + }.compact + end + end +end diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl new file mode 100644 index 00000000000..92511e26861 --- /dev/null +++ b/lib/support/nginx/registry-ssl @@ -0,0 +1,53 @@ +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### + +## Redirects all HTTP traffic to the HTTPS host +server { + listen *:80; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + return 301 https://$http_host:$request_uri; + access_log /var/log/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/nginx/gitlab_registry_error.log; +} + +server { + # If a different port is specified in https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/config/gitlab.yml.example#L182, + # it should be declared here as well + listen *:443 ssl http2; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + + client_max_body_size 0; + chunked_transfer_encoding on; + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt + ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key + + ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4'; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache builtin:1000 shared:SSL:10m; + ssl_session_timeout 5m; + + access_log /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/gitlab/nginx/gitlab_registry_error.log; + + location / { + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass http://localhost:5000; + } + +} diff --git a/lib/tasks/auto_annotate_models.rake b/lib/tasks/auto_annotate_models.rake deleted file mode 100644 index 16bad4bd2bd..00000000000 --- a/lib/tasks/auto_annotate_models.rake +++ /dev/null @@ -1,44 +0,0 @@ -if Rails.env.development? - task :set_annotation_options do - # You can override any of these by setting an environment variable of the - # same name. - Annotate.set_defaults( - 'routes' => 'false', - 'position_in_routes' => 'before', - 'position_in_class' => 'before', - 'position_in_test' => 'before', - 'position_in_fixture' => 'before', - 'position_in_factory' => 'before', - 'position_in_serializer' => 'before', - 'show_foreign_keys' => 'true', - 'show_indexes' => 'false', - 'simple_indexes' => 'false', - 'model_dir' => 'app/models', - 'root_dir' => '', - 'include_version' => 'false', - 'require' => '', - 'exclude_tests' => 'true', - 'exclude_fixtures' => 'true', - 'exclude_factories' => 'true', - 'exclude_serializers' => 'true', - 'exclude_scaffolds' => 'true', - 'exclude_controllers' => 'true', - 'exclude_helpers' => 'true', - 'ignore_model_sub_dir' => 'false', - 'ignore_columns' => nil, - 'ignore_unknown_models' => 'false', - 'hide_limit_column_types' => 'integer,boolean', - 'skip_on_db_migrate' => 'false', - 'format_bare' => 'true', - 'format_rdoc' => 'false', - 'format_markdown' => 'false', - 'sort' => 'false', - 'force' => 'false', - 'trace' => 'false', - 'wrapper_open' => nil, - 'wrapper_close' => nil, - ) - end - - Annotate.load_tasks -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 402bb338f27..9ee72fde92f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,6 +14,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke + Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -39,14 +40,14 @@ namespace :gitlab do removed. MSG ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.yellow + puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) end # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade - $progress.puts 'Cleaning the database ... '.blue + $progress.puts 'Cleaning the database ... '.color(:blue) Rake::Task['gitlab:db:drop_tables'].invoke - $progress.puts 'done'.green + $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') @@ -54,6 +55,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke backup.cleanup @@ -61,115 +63,142 @@ namespace :gitlab do namespace :repo do task create: :environment do - $progress.puts "Dumping repositories ...".blue + $progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Repository.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring repositories ...".blue + $progress.puts "Restoring repositories ...".color(:blue) Backup::Repository.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :db do task create: :environment do - $progress.puts "Dumping database ... ".blue + $progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Database.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring database ... ".blue + $progress.puts "Restoring database ... ".color(:blue) Backup::Database.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :builds do task create: :environment do - $progress.puts "Dumping builds ... ".blue + $progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Builds.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring builds ... ".blue + $progress.puts "Restoring builds ... ".color(:blue) Backup::Builds.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :uploads do task create: :environment do - $progress.puts "Dumping uploads ... ".blue + $progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Uploads.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring uploads ... ".blue + $progress.puts "Restoring uploads ... ".color(:blue) Backup::Uploads.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :artifacts do task create: :environment do - $progress.puts "Dumping artifacts ... ".blue + $progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Artifacts.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring artifacts ... ".blue + $progress.puts "Restoring artifacts ... ".color(:blue) Backup::Artifacts.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) end end namespace :lfs do task create: :environment do - $progress.puts "Dumping lfs objects ... ".blue + $progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - $progress.puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".color(:cyan) else Backup::Lfs.new.dump - $progress.puts "done".green + $progress.puts "done".color(:green) end end task restore: :environment do - $progress.puts "Restoring lfs objects ... ".blue + $progress.puts "Restoring lfs objects ... ".color(:blue) Backup::Lfs.new.restore - $progress.puts "done".green + $progress.puts "done".color(:green) + end + end + + namespace :registry do + task create: :environment do + $progress.puts "Dumping container registry images ... ".color(:blue) + + if Gitlab.config.registry.enabled + if ENV["SKIP"] && ENV["SKIP"].include?("registry") + $progress.puts "[SKIPPED]".color(:cyan) + else + Backup::Registry.new.dump + $progress.puts "done".color(:green) + end + else + $progress.puts "[DISABLED]".color(:cyan) + end + end + + task restore: :environment do + $progress.puts "Restoring container registry images ... ".color(:blue) + if Gitlab.config.registry.enabled + Backup::Registry.new.restore + $progress.puts "done".color(:green) + else + $progress.puts "[DISABLED]".color(:cyan) + end end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index effb8eb6001..12d6ac45fb6 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -50,14 +50,14 @@ namespace :gitlab do end if correct_options.all? - puts "yes".green + puts "yes".color(:green) else print "Trying to fix Git error automatically. ..." if auto_fix_git_config(options) - puts "Success".green + puts "Success".color(:green) else - puts "Failed".red + puts "Failed".color(:red) try_fixing_it( sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") ) @@ -74,9 +74,9 @@ namespace :gitlab do database_config_file = Rails.root.join("config", "database.yml") if File.exists?(database_config_file) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Copy config/database.yml.<your db> to config/database.yml", "Check that the information in config/database.yml is correct" @@ -95,9 +95,9 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") if File.exists?(gitlab_config_file) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Copy config/gitlab.yml.example to config/gitlab.yml", "Update config/gitlab.yml to match your setup" @@ -114,14 +114,14 @@ namespace :gitlab do gitlab_config_file = Rails.root.join("config", "gitlab.yml") unless File.exists?(gitlab_config_file) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) end # omniauth or ldap could have been deleted from the file unless Gitlab.config['git_host'] - puts "no".green + puts "no".color(:green) else - puts "yes".red + puts "yes".color(:red) try_fixing_it( "Backup your config/gitlab.yml", "Copy config/gitlab.yml.example to config/gitlab.yml", @@ -138,16 +138,16 @@ namespace :gitlab do print "Init script exists? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end script_path = "/etc/init.d/gitlab" if File.exists?(script_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Install the init script" ) @@ -162,7 +162,7 @@ namespace :gitlab do print "Init script up-to-date? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end @@ -170,7 +170,7 @@ namespace :gitlab do script_path = "/etc/init.d/gitlab" unless File.exists?(script_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end @@ -178,9 +178,9 @@ namespace :gitlab do script_content = File.read(script_path) if recipe_content == script_content - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Redownload the init script" ) @@ -197,9 +197,9 @@ namespace :gitlab do migration_status, _ = Gitlab::Popen.popen(%W(bundle exec rake db:migrate:status)) unless migration_status =~ /down\s+\d{14}/ - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") ) @@ -210,13 +210,13 @@ namespace :gitlab do def check_orphaned_group_members print "Database contains orphaned GroupMembers? ... " if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".red + puts "yes".color(:red) try_fixing_it( "You can delete the orphaned records using something along the lines of:", sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") ) else - puts "no".green + puts "no".color(:green) end end @@ -226,9 +226,9 @@ namespace :gitlab do log_path = Rails.root.join("log") if File.writable?(log_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R gitlab #{log_path}", "sudo chmod -R u+rwX #{log_path}" @@ -246,9 +246,9 @@ namespace :gitlab do tmp_path = Rails.root.join("tmp") if File.writable?(tmp_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R gitlab #{tmp_path}", "sudo chmod -R u+rwX #{tmp_path}" @@ -264,7 +264,7 @@ namespace :gitlab do print "Uploads directory setup correctly? ... " unless File.directory?(Rails.root.join('public/uploads')) - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" ) @@ -280,16 +280,16 @@ namespace :gitlab do if File.stat(upload_path).mode == 040700 unless Dir.exists?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.magenta + puts 'skipped (no tmp uploads folder yet)'.color(:magenta) return end # If tmp upload dir has incorrect permissions, assume others do as well # Verify drwx------ permissions if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chown -R #{gitlab_user} #{upload_path}", "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", @@ -301,9 +301,9 @@ namespace :gitlab do fix_and_rerun end else - puts "no".red + puts "no".color(:red) try_fixing_it( - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" + "sudo chmod 700 #{upload_path}" ) for_more_information( see_installation_guide_section "GitLab" @@ -320,9 +320,9 @@ namespace :gitlab do redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your redis server to a version >= #{min_redis_version}" ) @@ -361,10 +361,10 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path if File.exists?(repo_base_path) - puts "yes".green + puts "yes".color(:green) else - puts "no".red - puts "#{repo_base_path} is missing".red + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) try_fixing_it( "This should have been created when setting up GitLab Shell.", "Make sure it's set correctly in config/gitlab.yml", @@ -382,14 +382,14 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end unless File.symlink?(repo_base_path) - puts "no".green + puts "no".color(:green) else - puts "yes".red + puts "yes".color(:red) try_fixing_it( "Make sure it's set to the real directory in config/gitlab.yml" ) @@ -402,14 +402,14 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", @@ -429,17 +429,17 @@ namespace :gitlab do repo_base_path = Gitlab.config.gitlab_shell.repos_path unless File.exists?(repo_base_path) - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end uid = uid_for(gitlab_shell_ssh_user) gid = gid_for(gitlab_shell_owner_group) if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".green + puts "yes".color(:green) else - puts "no".red - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".blue + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) try_fixing_it( "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" ) @@ -456,7 +456,7 @@ namespace :gitlab do gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path unless Project.count > 0 - puts "can't check, you have no projects".magenta + puts "can't check, you have no projects".color(:magenta) return end puts "" @@ -466,12 +466,12 @@ namespace :gitlab do project_hook_directory = File.join(project.repository.path_to_repo, "hooks") if project.empty_repo? - puts "repository is empty".magenta + puts "repository is empty".color(:magenta) elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) && (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path)) - puts 'ok'.green + puts 'ok'.color(:green) else - puts "wrong or missing hooks".red + puts "wrong or missing hooks".color(:red) try_fixing_it( sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), 'Check the hooks_path in config/gitlab.yml', @@ -491,9 +491,9 @@ namespace :gitlab do check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) puts "Running #{check_cmd}" if system(check_cmd, chdir: gitlab_shell_repo_base) - puts 'gitlab-shell self-check successful'.green + puts 'gitlab-shell self-check successful'.color(:green) else - puts 'gitlab-shell self-check failed'.red + puts 'gitlab-shell self-check failed'.color(:red) try_fixing_it( 'Make sure GitLab is running;', 'Check the gitlab-shell configuration file:', @@ -507,7 +507,7 @@ namespace :gitlab do print "projects have namespace: ... " unless Project.count > 0 - puts "can't check, you have no projects".magenta + puts "can't check, you have no projects".color(:magenta) return end puts "" @@ -516,9 +516,9 @@ namespace :gitlab do print sanitized_message(project) if project.namespace - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Migrate global projects" ) @@ -576,9 +576,9 @@ namespace :gitlab do print "Running? ... " if sidekiq_process_count > 0 - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("RAILS_ENV=production bin/background_jobs start") ) @@ -596,9 +596,9 @@ namespace :gitlab do print 'Number of Sidekiq processes ... ' if process_count == 1 - puts '1'.green + puts '1'.color(:green) else - puts "#{process_count}".red + puts "#{process_count}".color(:red) try_fixing_it( 'sudo service gitlab stop', "sudo pkill -u #{gitlab_user} -f sidekiq", @@ -646,16 +646,16 @@ namespace :gitlab do print "Init.d configured correctly? ... " if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.magenta + puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) return end path = "/etc/default/gitlab" if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Enable mail_room in the init.d configuration." ) @@ -672,9 +672,9 @@ namespace :gitlab do path = Rails.root.join("Procfile") if File.exist?(path) && File.read(path) =~ /^mail_room:/ - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Enable mail_room in your Procfile." ) @@ -691,14 +691,14 @@ namespace :gitlab do path = "/etc/default/gitlab" unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "can't check because of previous errors".magenta + puts "can't check because of previous errors".color(:magenta) return end if mail_room_running? - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( sudo_gitlab("RAILS_ENV=production bin/mail_room start") ) @@ -729,9 +729,9 @@ namespace :gitlab do end if connected - puts "yes".green + puts "yes".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Check that the information in config/gitlab.yml is correct" ) @@ -799,7 +799,7 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".blue) + username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| @@ -811,7 +811,7 @@ namespace :gitlab do repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } else - puts "\nUser '#{username}' not found".red + puts "\nUser '#{username}' not found".color(:red) end end end @@ -820,13 +820,13 @@ namespace :gitlab do ########################## def fix_and_rerun - puts " Please #{"fix the error above"} and rerun the checks.".red + puts " Please #{"fix the error above"} and rerun the checks.".color(:red) end def for_more_information(*sources) sources = sources.shift if sources.first.is_a?(Array) - puts " For more information see:".blue + puts " For more information see:".color(:blue) sources.each do |source| puts " #{source}" end @@ -834,7 +834,7 @@ namespace :gitlab do def finished_checking(component) puts "" - puts "Checking #{component.yellow} ... #{"Finished".green}" + puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" puts "" end @@ -855,14 +855,14 @@ namespace :gitlab do end def start_checking(component) - puts "Checking #{component.yellow} ..." + puts "Checking #{component.color(:yellow)} ..." puts "" end def try_fixing_it(*steps) steps = steps.shift if steps.first.is_a?(Array) - puts " Try fixing it:".blue + puts " Try fixing it:".color(:blue) steps.each do |step| puts " #{step}" end @@ -874,9 +874,9 @@ namespace :gitlab do print "GitLab Shell version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "OK (#{current_version})".green + puts "OK (#{current_version})".color(:green) else - puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".red + puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end @@ -887,9 +887,9 @@ namespace :gitlab do print "Ruby version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".green + puts "yes (#{current_version})".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your ruby to a version >= #{required_version} from #{current_version}" ) @@ -905,9 +905,9 @@ namespace :gitlab do print "Git version >= #{required_version} ? ... " if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".green + puts "yes (#{current_version})".color(:green) else - puts "no".red + puts "no".color(:red) try_fixing_it( "Update your git to a version >= #{required_version} from #{current_version}" ) @@ -925,9 +925,9 @@ namespace :gitlab do def sanitized_message(project) if should_sanitize? - "#{project.namespace_id.to_s.yellow}/#{project.id.to_s.yellow} ... " + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " else - "#{project.name_with_namespace.yellow} ... " + "#{project.name_with_namespace.color(:yellow)} ... " end end @@ -940,7 +940,7 @@ namespace :gitlab do end def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.yellow}" + puts "\nChecking repo at #{repo_dir.color(:yellow)}" git_fsck(repo_dir) check_config_lock(repo_dir) @@ -948,25 +948,25 @@ namespace :gitlab do end def git_fsck(repo_dir) - puts "Running `git fsck`".yellow + puts "Running `git fsck`".color(:yellow) system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) end def check_config_lock(repo_dir) config_exists = File.exist?(File.join(repo_dir,'config.lock')) - config_output = config_exists ? 'yes'.red : 'no'.green - puts "'config.lock' file exists?".yellow + " ... #{config_output}" + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" end def check_ref_locks(repo_dir) lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) if lock_files.present? - puts "Ref lock files exist:".red + puts "Ref lock files exist:".color(:red) lock_files.each do |lock_file| puts " #{lock_file}" end else - puts "No ref lock files exist".green + puts "No ref lock files exist".color(:green) end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 9f5852ac613..ab0028d6603 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -10,7 +10,7 @@ namespace :gitlab do git_base_path = Gitlab.config.gitlab_shell.repos_path all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.yellow + puts git_base_path.color(:yellow) puts "Looking for directories to remove... " all_dirs.reject! do |dir| @@ -29,17 +29,17 @@ namespace :gitlab do if remove_flag if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".red + puts "Removed...#{dir_path}".color(:red) else - puts "Cannot remove #{dir_path}".red + puts "Cannot remove #{dir_path}".color(:red) end else - puts "Can be removed: #{dir_path}".red + puts "Can be removed: #{dir_path}".color(:red) end end unless remove_flag - puts "To cleanup this directories run this command with REMOVE=true".yellow + puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) end end @@ -75,19 +75,19 @@ namespace :gitlab do next unless user.ldap_user? print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." if Gitlab::LDAP::Access.allowed?(user) - puts " [OK]".green + puts " [OK]".color(:green) else if block_flag user.block! unless user.blocked? - puts " [BLOCKED]".red + puts " [BLOCKED]".color(:red) else - puts " [NOT IN LDAP]".yellow + puts " [NOT IN LDAP]".color(:yellow) end end end unless block_flag - puts "To block these users run this command with BLOCK=true".yellow + puts "To block these users run this command with BLOCK=true".color(:yellow) end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 1c706dc11b3..7230b9485be 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -3,22 +3,22 @@ namespace :gitlab do desc 'GitLab | Manually insert schema migration version' task :mark_migration_complete, [:version] => :environment do |_, args| unless args[:version] - puts "Must specify a migration version as an argument".red + puts "Must specify a migration version as an argument".color(:red) exit 1 end version = args[:version].to_i if version == 0 - puts "Version '#{args[:version]}' must be a non-zero integer".red + puts "Version '#{args[:version]}' must be a non-zero integer".color(:red) exit 1 end sql = "INSERT INTO schema_migrations (version) VALUES (#{version})" begin ActiveRecord::Base.connection.execute(sql) - puts "Successfully marked '#{version}' as complete".green + puts "Successfully marked '#{version}' as complete".color(:green) rescue ActiveRecord::RecordNotUnique - puts "Migration version '#{version}' is already marked complete".yellow + puts "Migration version '#{version}' is already marked complete".color(:yellow) end end @@ -29,10 +29,22 @@ namespace :gitlab do tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run connection.execute('TRUNCATE schema_migrations') + # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html - tables.each { |t| connection.execute("DROP TABLE #{t} CASCADE") } + # Add `IF EXISTS` because cascade could have already deleted a table. + tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + end + + desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' + task configure: :environment do + if ActiveRecord::Base.connection.tables.any? + Rake::Task['db:migrate'].invoke + else + Rake::Task['db:schema:load'].invoke + Rake::Task['db:seed_fu'].invoke + end end end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 65ee430d550..f9834a4dae8 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -5,7 +5,7 @@ namespace :gitlab do task repack: :environment do failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -15,7 +15,7 @@ namespace :gitlab do task gc: :environment do failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -25,7 +25,7 @@ namespace :gitlab do task prune: :environment do failures = perform_git_cmd(%W(git prune), "Git Prune") if failures.empty? - puts "Done".green + puts "Done".color(:green) else output_failures(failures) end @@ -47,7 +47,7 @@ namespace :gitlab do end def output_failures(failures) - puts "The following repositories reported errors:".red + puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 1c04f47f08f..4753f00c26a 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -23,7 +23,7 @@ namespace :gitlab do group_name, name = File.split(path) group_name = nil if group_name == '.' - puts "Processing #{repo_path}".yellow + puts "Processing #{repo_path}".color(:yellow) if path.end_with?('.wiki') puts " * Skipping wiki repo" @@ -51,9 +51,9 @@ namespace :gitlab do group.path = group_name group.owner = user if group.save - puts " * Created Group #{group.name} (#{group.id})".green + puts " * Created Group #{group.name} (#{group.id})".color(:green) else - puts " * Failed trying to create group #{group.name}".red + puts " * Failed trying to create group #{group.name}".color(:red) end end # set project group @@ -63,17 +63,17 @@ namespace :gitlab do project = Projects::CreateService.new(user, project_params).execute if project.persisted? - puts " * Created #{project.name} (#{repo_path})".green + puts " * Created #{project.name} (#{repo_path})".color(:green) project.update_repository_size project.update_commit_count else - puts " * Failed trying to create #{project.name} (#{repo_path})".red - puts " Errors: #{project.errors.messages}".red + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) end end end - puts "Done!".green + puts "Done!".color(:green) end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index d6883a563ee..352b566df24 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -15,15 +15,15 @@ namespace :gitlab do rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) puts "" - puts "System information".yellow - puts "System:\t\t#{os_name || "unknown".red}" + puts "System information".color(:yellow) + puts "System:\t\t#{os_name || "unknown".color(:red)}" puts "Current User:\t#{run(%W(whoami))}" - puts "Using RVM:\t#{rvm_version.present? ? "yes".green : "no"}" + puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? - puts "Ruby Version:\t#{ruby_version || "unknown".red}" - puts "Gem Version:\t#{gem_version || "unknown".red}" - puts "Bundler Version:#{bunder_version || "unknown".red}" - puts "Rake Version:\t#{rake_version || "unknown".red}" + puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" + puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" + puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" + puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" @@ -39,7 +39,7 @@ namespace :gitlab do omniauth_providers.map! { |provider| provider['name'] } puts "" - puts "GitLab information".yellow + puts "GitLab information".color(:yellow) puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab::REVISION}" puts "Directory:\t#{Rails.root}" @@ -47,9 +47,9 @@ namespace :gitlab do puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" - puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".green : "no"}" - puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".green : "no"}" - puts "Omniauth Providers: #{omniauth_providers.map(&:magenta).join(', ')}" if Gitlab.config.omniauth.enabled + puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}" + puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}" + puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled @@ -60,8 +60,8 @@ namespace :gitlab do end puts "" - puts "GitLab Shell".yellow - puts "Version:\t#{gitlab_shell_version || "unknown".red}" + puts "GitLab Shell".color(:yellow) + puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 48baecfd2a2..05fcb8e3da5 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -19,7 +19,7 @@ namespace :gitlab do Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) exit 1 end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd61632e557..b1648a4602a 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -118,12 +118,12 @@ namespace :gitlab do puts "" unless $?.success? - puts "Failed to add keys...".red + puts "Failed to add keys...".color(:red) exit 1 end rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) exit 1 end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d33b5b31e18..d0c019044b7 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,7 +2,7 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -String.disable_colorization = true unless STDOUT.isatty +require 'rainbow/ext/string' # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] @@ -14,7 +14,7 @@ namespace :gitlab do # Returns "yes" the user chose to continue # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".blue, %w{yes no}) + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) raise Gitlab::TaskAbortedByUserError unless answer == "yes" end @@ -98,10 +98,10 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts " Warning ".colorize(:black).on_yellow - puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.magenta}." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." puts "" end @warned_user_not_gitlab = true diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 9196677a017..fc0ccc726ed 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -6,17 +6,17 @@ namespace :gitlab do count = scope.count if count > 0 - puts "This will disable 2FA for #{count.to_s.red} users..." + puts "This will disable 2FA for #{count.to_s.color(:red)} users..." begin ask_to_continue scope.find_each(&:disable_two_factor!) - puts "Successfully disabled 2FA for #{count} users.".green + puts "Successfully disabled 2FA for #{count} users.".color(:green) rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".red + puts "Quitting...".color(:red) end else - puts "There are currently no users with 2FA enabled.".yellow + puts "There are currently no users with 2FA enabled.".color(:yellow) end end end diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake index 9b636f12d9f..3bd10b0208b 100644 --- a/lib/tasks/gitlab/update_commit_count.rake +++ b/lib/tasks/gitlab/update_commit_count.rake @@ -6,15 +6,15 @@ namespace :gitlab do ask_to_continue unless ENV['force'] == 'yes' projects.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.yellow} ... " + print "#{project.name_with_namespace.color(:yellow)} ... " unless project.repo_exists? - puts "skipping, because the repo is empty".magenta + puts "skipping, because the repo is empty".color(:magenta) next end project.update_commit_count - puts project.commit_count.to_s.green + puts project.commit_count.to_s.color(:green) end end end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake new file mode 100644 index 00000000000..4fd48cccb1d --- /dev/null +++ b/lib/tasks/gitlab/update_gitignore.rake @@ -0,0 +1,46 @@ +namespace :gitlab do + desc "GitLab | Update gitignore" + task :update_gitignore do + unless clone_gitignores + puts "Cloning the gitignores failed".color(:red) + return + end + + remove_unneeded_files(gitignore_directory) + remove_unneeded_files(global_directory) + + puts "Done".color(:green) + end + + def clone_gitignores + FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) + FileUtils.cd vendor_directory + + system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') + end + + # Retain only certain files: + # - The LICENSE, because we have to + # - The sub dir global + # - The gitignores themself + # - Dir.entires returns also the entries '.' and '..' + def remove_unneeded_files(path) + Dir.foreach(path) do |file| + FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ + end + end + + private + + def vendor_directory + Rails.root.join('vendor') + end + + def gitignore_directory + File.join(vendor_directory, 'gitignore') + end + + def global_directory + File.join(gitignore_directory, 'Global') + end +end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index cc0f668474e..f467cc0ee29 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -12,9 +12,9 @@ namespace :gitlab do print "- #{project.name} ... " web_hook = project.hooks.new(url: web_hook_url) if web_hook.save - puts "added".green + puts "added".color(:green) else - print "failed".red + print "failed".color(:red) puts " [#{web_hook.errors.full_messages.to_sentence}]" end end @@ -57,7 +57,7 @@ namespace :gitlab do if namespace Project.in_namespace(namespace.id) else - puts "Namespace not found: #{namespace_path}".red + puts "Namespace not found: #{namespace_path}".color(:red) exit 2 end end diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index d258c6fd08d..4f2486157b7 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -1,6 +1,6 @@ desc "GitLab | Build internal ids for issues and merge requests" task migrate_iids: :environment do - puts 'Issues'.yellow + puts 'Issues'.color(:yellow) Issue.where(iid: nil).find_each(batch_size: 100) do |issue| begin issue.set_iid @@ -15,7 +15,7 @@ task migrate_iids: :environment do end puts 'done' - puts 'Merge Requests'.yellow + puts 'Merge Requests'.color(:yellow) MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr| begin mr.set_iid @@ -30,7 +30,7 @@ task migrate_iids: :environment do end puts 'done' - puts 'Milestones'.yellow + puts 'Milestones'.color(:yellow) Milestone.where(iid: nil).find_each(batch_size: 100) do |m| begin m.set_iid diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index ddfaf5d51f2..78ffccc9d06 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,4 +1,5 @@ unless Rails.env.production? require 'rubocop/rake_task' + RuboCop::RakeTask.new end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 01d23b89bb7..da255f5464b 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -52,7 +52,7 @@ def run_spinach_tests(tags) tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) puts '' - puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red + puts "Spinach tests for #{tags}: Retrying tests... #{tests}".color(:red) puts '' sleep(3) success = run_spinach_command(tests) |