diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-01-04 22:25:55 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-01-04 22:25:55 +0800 |
| commit | 104bac3d215383b76b058e8f61b90fdfac936341 (patch) | |
| tree | 5b0737050878d35e0963272810637e83350ce696 /lib | |
| parent | 99b556976370bfe0c052d15b6a8f0642256173fd (diff) | |
| parent | 034d2e4e749ee09649062d3fb1d26c53021ab4d8 (diff) | |
| download | gitlab-ce-104bac3d215383b76b058e8f61b90fdfac936341.tar.gz | |
Merge branch 'master' into fix-git-hooks-when-creating-file
* master: (1031 commits)
Add changelog entry for renaming API param [ci skip]
Add missing milestone parameter
Refactor issues filter in API
Fix project hooks params
Gitlab::LDAP::Person uses LDAP attributes configuration
Don't delete files from spec/fixtures
Copy, don't move uploaded avatar files
Minor improvements to changelog docs
Rename logo, apply for Slack too
Fix Gemfile.lock for the octokit update
Fix cross-project references copy to include the project reference
Add logo in public files
Use stable icon for Mattermost integration
rewrite the item.respond_to?(:x?) && item.x? to item.try(:x?)
API: extern_uid is a string
Increases pipeline graph drowdown width in order to prevent strange position on chrome on ubuntu
Removed bottom padding from merge manually from CLI because of repositioning award emoji's
Make haml_lint happy
Improve spec
Add feature tests for Cycle Analytics
...
Diffstat (limited to 'lib')
123 files changed, 3282 insertions, 914 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index cec2702e44d..9d5adffd8f4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,8 @@ module API include APIGuard version 'v3', using: :path + before { allow_access_with_scope :api } + rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8cc7a26f1fa..df6db140d0e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -44,27 +47,60 @@ module API access_token = find_access_token return nil unless access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED + when AccessTokenValidationService::EXPIRED raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED + when AccessTokenValidationService::REVOKED raise RevokedError - when Oauth2::AccessTokenValidationService::VALID + when AccessTokenValidationService::VALID @current_user = User.find(access_token.resource_owner_id) end end + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + + return nil unless token_string.present? + + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + end + def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -72,10 +108,6 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2670a2d413a..cf2489dbb67 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,7 +1,6 @@ require 'mime/types' module API - # Projects commits API class Commits < Grape::API include PaginationParams @@ -121,6 +120,41 @@ module API present paginate(notes), with: Entities::CommitNote end + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + create_merge_request: false, + source_project: user_project, + source_branch: commit.cherry_pick_branch_name, + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end + desc 'Post comment to commit' do success Entities::CommitNote end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 01c0f5072ba..d2fadf6a3d0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -78,21 +78,21 @@ module API expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible - expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) } - expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) } - expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) } - expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) } - expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) } + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } expose :created_at, :last_activity_at expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace + expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| @@ -101,6 +101,16 @@ module API expose :only_allow_merge_if_build_succeeds expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved + + expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size end class Member < UserBasic @@ -127,6 +137,15 @@ module API expose :avatar_url expose :web_url expose :request_access_enabled + + expose :statistics, if: :statistics do + with_options format_with: -> (value) { value.to_i } do + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + end end class GroupDetail < Group @@ -298,7 +317,7 @@ module API end class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at + expose :id, :title, :key, :created_at, :can_push end class SSHKeyWithUser < SSHKey @@ -391,7 +410,7 @@ module API end class Namespace < Grape::Entity - expose :id, :path, :kind + expose :id, :name, :path, :kind end class MemberAccess < Grape::Entity @@ -440,12 +459,12 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:user].id) + project.project_members.find_by(user_id: options[:current_user].id) end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - project.group.group_members.find_by(user_id: options[:user].id) + project.group.group_members.find_by(user_id: options[:current_user].id) end end end @@ -629,7 +648,7 @@ module API end class EnvironmentBasic < Grape::Entity - expose :id, :name, :external_url + expose :id, :name, :slug, :external_url end class Environment < EnvironmentBasic diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 80bbd9bb6e4..1a7e68f0528 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,6 +1,7 @@ module API # Environments RESTfull API endpoints class Environments < Grape::API + include ::API::Helpers::CustomValidators include PaginationParams before { authenticate! } @@ -29,6 +30,7 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end post ':id/environments' do authorize! :create_environment, user_project @@ -50,6 +52,7 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' optional :name, type: String, desc: 'The new environment name' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end put ':id/environments/:environment_id' do authorize! :update_environment, user_project diff --git a/lib/api/files.rb b/lib/api/files.rb index 28f306e45f3..2e79e22e649 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,8 +1,6 @@ module API # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { @@ -70,7 +68,7 @@ module API ref: params[:ref], blob_id: blob.id, commit_id: commit.id, - last_commit_id: repo.last_commit_for_path(commit.sha, params[:file_path]).id + last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) } end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 105d3ee342e..e04d2e40fb6 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -11,6 +11,20 @@ module API optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end end resource :groups do @@ -18,6 +32,7 @@ module API success Entities::Group end params do + use :statistics_params optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' @@ -38,7 +53,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present paginate(groups), with: Entities::Group + present_groups groups, statistics: params[:statistics] && current_user.is_admin? end desc 'Get list of owned groups for authenticated user' do @@ -46,10 +61,10 @@ module API end params do use :pagination + use :statistics_params end get '/owned' do - groups = current_user.owned_groups - present paginate(groups), with: Entities::Group, user: current_user + present_groups current_user.owned_groups, statistics: params[:statistics] end desc 'Create a group. Available only for users who can create groups.' do @@ -66,7 +81,7 @@ module API group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute if group.persisted? - present group, with: Entities::Group + present group, with: Entities::Group, current_user: current_user else render_api_error!("Failed to save group #{group.errors.messages}", 400) end @@ -92,7 +107,7 @@ module API authorize! :admin_group, group if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute - present group, with: Entities::GroupDetail + present group, with: Entities::GroupDetail, current_user: current_user else render_validation_error!(group) end @@ -103,7 +118,7 @@ module API end get ":id" do group = find_group!(params[:id]) - present group, with: Entities::GroupDetail + present group, with: Entities::GroupDetail, current_user: current_user end desc 'Remove a group.' @@ -125,13 +140,16 @@ module API default: 'created_at', desc: 'Return projects ordered by field' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return projects sorted in ascending and descending order' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' use :pagination end get ":id/projects" do group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) projects = filter_projects(projects) - present paginate(projects), with: Entities::Project, user: current_user + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project + present paginate(projects), with: entity, current_user: current_user end desc 'Transfer a project to the group namespace. Available only for admin.' do @@ -147,7 +165,7 @@ module API result = ::Projects::TransferService.new(project, current_user).execute(group) if result - present group, with: Entities::GroupDetail + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to transfer project #{project.errors.messages}", 400) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8b0f8deadfa..ee9247ee240 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,72 +2,26 @@ module API module Helpers include Gitlab::Utils - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def private_token - params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] - end - - def warden - env['warden'] - end - - # Check the Rails session for valid authentication details - # - # Until CSRF protection is added to the API, disallow this method for - # state-changing endpoints - def find_user_from_warden - warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) - end - def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) declared(params, options).to_h.symbolize_keys end - def find_user_by_private_token - token = private_token - return nil unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - def current_user - @current_user ||= find_user_by_private_token - @current_user ||= doorkeeper_guard - @current_user ||= find_user_from_warden + return @current_user if defined?(@current_user) - unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? - return nil - end - - identifier = sudo_identifier + @current_user = initial_current_user - if identifier - # We check for private_token because we cannot allow PAT to be used - forbidden!('Must be admin to use sudo') unless @current_user.is_admin? - forbidden!('Private token must be specified in order to use sudo') unless private_token_used? - - @impersonator = @current_user - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end + sudo! @current_user end - def sudo_identifier - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /\A[0-9]+\z/) - identifier.to_i - else - identifier - end + def sudo? + initial_current_user != current_user end def user_project @@ -78,6 +32,14 @@ module API @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end + def find_user(id) + if id =~ /^\d+$/ + User.find_by(id: id) + else + User.find_by(username: id) + end + end + def find_project(id) if id =~ /^\d+$/ Project.find_by(id: id) @@ -96,17 +58,6 @@ module API end end - def project_service(project = user_project) - @project_service ||= project.find_or_initialize_service(params[:service_slug].underscore) - @project_service || not_found!("Service") - end - - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym - end - end - def find_group(id) if id =~ /^\d+$/ Group.find_by(id: id) @@ -145,7 +96,7 @@ module API end def authenticate_non_get! - authenticate! unless %w[GET HEAD].include?(route.route_method) + authenticate! unless %w[GET HEAD].include?(route.request_method) end def authenticate_by_gitlab_shell_token! @@ -297,7 +248,7 @@ module API rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) end - # Projects helpers + # project helpers def filter_projects(projects) if params[:search].present? @@ -354,6 +305,62 @@ module API private + def private_token + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints + def find_user_from_warden + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) + end + + def initial_current_user + return @initial_current_user if defined?(@initial_current_user) + + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden + + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end + + @initial_current_user + end + + def sudo! + return unless sudo_identifier + return unless initial_current_user + + unless initial_current_user.is_admin? + forbidden!('Must be admin to use sudo') + end + + # Only private tokens should be used for the SUDO feature + unless private_token == initial_current_user.private_token + forbidden!('Private token must be specified in order to use sudo') + end + + sudoed_user = find_user(sudo_identifier) + + if sudoed_user + @current_user = sudoed_user + else + not_found!("No user id or username for: #{sudo_identifier}") + end + end + + def sudo_identifier + @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] + end + def add_pagination_headers(paginated_data) header 'X-Total', paginated_data.total_count.to_s header 'X-Total-Pages', paginated_data.total_pages.to_s @@ -386,10 +393,6 @@ module API links.join(', ') end - def private_token_used? - private_token == @current_user.private_token - end - def secret_token Gitlab::Shell.secret_token end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb new file mode 100644 index 00000000000..0a8f3073a50 --- /dev/null +++ b/lib/api/helpers/custom_validators.rb @@ -0,0 +1,14 @@ +module API + module Helpers + module CustomValidators + class Absence < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) + end + end + end + end +end + +Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eb223c1101d..e8975eb57e0 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -52,6 +52,14 @@ module API :push_code ] end + + def parse_allowed_environment_variables + return if params[:env].blank? + + JSON.parse(params[:env]) + + rescue JSON::ParserError + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 7087ce11401..db2d18f935d 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -32,7 +32,11 @@ module API if wiki? Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + Gitlab::GitAccess.new(actor, + project, + protocol, + authentication_abilities: ssh_authentication_abilities, + env: parse_allowed_environment_variables) end access_status = access.check(params[:action], params[:changes]) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c9124649cbb..54b97402426 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -5,28 +5,18 @@ module API before { authenticate! } helpers do - def filter_issues_state(issues, state) - case state - when 'opened' then issues.opened - when 'closed' then issues.closed - else issues - end - end - + # TODO: Remove in 9.0 and switch to IssueFinder-based label filtering def filter_issues_labels(issues, labels) issues.includes(:labels).where('labels.title' => labels.split(',')) end - def filter_issues_milestone(issues, milestone) - issues.includes(:milestone).where('milestones.title' => milestone) - end - params :issues_params do optional :labels, type: String, desc: 'Comma-separated list of label names' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return issues ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return issues sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return issues for a specific milestone' use :pagination end @@ -37,8 +27,6 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' - optional :state_event, type: String, values: %w[open close], - desc: 'State of the issue' end end @@ -52,8 +40,7 @@ module API use :issues_params end get do - issues = current_user.issues.inc_notes_with_associations - issues = filter_issues_state(issues, params[:state]) + issues = IssuesFinder.new(current_user, scope: 'all', author_id: current_user.id, state: params[:state]).execute.inc_notes_with_associations issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = issues.reorder(params[:order_by] => params[:sort]) @@ -101,16 +88,14 @@ module API use :issues_params end get ":id/issues" do - issues = IssuesFinder.new(current_user, project_id: user_project.id).execute.inc_notes_with_associations - issues = filter_issues_state(issues, params[:state]) + issues = IssuesFinder.new(current_user, + project_id: user_project.id, + state: params[:state], + milestone_title: params[:milestone]).execute.inc_notes_with_associations issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? - - unless params[:milestone].nil? - issues = filter_issues_milestone(issues, params[:milestone]) - end - issues = issues.reorder(params[:order_by] => params[:sort]) + present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end @@ -172,6 +157,7 @@ module API optional :title, type: String, desc: 'The title of an issue' optional :updated_at, type: DateTime, desc: 'Date time when the issue was updated. Available only for admins and project owners.' + optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params at_least_one_of :title, :description, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 55bdbc6a47c..5d1fe22f2df 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -143,8 +143,8 @@ module API success Entities::MergeRequest end params do - optional :title, type: String, desc: 'The title of the merge request' - optional :target_branch, type: String, desc: 'The target branch' + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen merge], desc: 'Status of the merge request' use :optional_params diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d0faf17714b..284e4cf549a 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -69,8 +69,6 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_str}/:noteable_id/notes" do - required_attributes! [:body] - opts = { note: params[:body], noteable_type: noteables_str.classify, diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index dcc0fb7a911..cb679e6658a 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -15,7 +15,7 @@ module API optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" optional :build_events, type: Boolean, desc: "Trigger hook on build events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" - optional :wiki_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2929d2157dc..3be14e8eb76 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -40,6 +40,15 @@ module API resource :projects do helpers do + params :collection_params do + use :sort_params + use :filter_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + params :sort_params do optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], default: 'created_at', desc: 'Return projects ordered by field' @@ -52,97 +61,94 @@ module API optional :visibility, type: String, values: %w[public internal private], desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' - use :sort_params + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options + end end desc 'Get a list of visible projects for authenticated user' do success Entities::BasicProjectDetails end params do - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - use :filter_params - use :pagination + use :collection_params end get '/visible' do - projects = ProjectsFinder.new.execute(current_user) - projects = filter_projects(projects) - entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess - - present paginate(projects), with: entity, user: current_user + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity end desc 'Get a projects list for authenticated user' do success Entities::BasicProjectDetails end params do - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - use :filter_params - use :pagination + use :collection_params end get do authenticate! - projects = current_user.authorized_projects - projects = filter_projects(projects) - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess - - present paginate(projects), with: entity, user: current_user + present_projects current_user.authorized_projects, + with: Entities::ProjectWithAccess end desc 'Get an owned projects list for authenticated user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params + use :statistics_params end get '/owned' do authenticate! - projects = current_user.owned_projects - projects = filter_projects(projects) - - present paginate(projects), with: Entities::ProjectWithAccess, user: current_user + present_projects current_user.owned_projects, + with: Entities::ProjectWithAccess, + statistics: params[:statistics] end desc 'Gets starred project for the authenticated user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params end get '/starred' do authenticate! - projects = current_user.viewable_starred_projects - projects = filter_projects(projects) - - present paginate(projects), with: Entities::Project, user: current_user + present_projects current_user.viewable_starred_projects end desc 'Get all projects for admin user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params + use :statistics_params end get '/all' do authenticated_as_admin! - projects = Project.all - projects = filter_projects(projects) - - present paginate(projects), with: Entities::ProjectWithAccess, user: current_user + present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics] end desc 'Search for projects the current user has access to' do @@ -221,7 +227,7 @@ module API end get ":id" do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present user_project, with: entity, user: current_user, + present user_project, with: entity, current_user: current_user, user_can_admin_project: can?(current_user, :admin_project, user_project) end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c287ee34a68..4ca6646a6f1 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -2,7 +2,6 @@ require 'mime/types' module API class Repositories < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do @@ -79,8 +78,6 @@ module API optional :format, type: String, desc: 'The archive format' end get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -96,7 +93,6 @@ module API requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' end get ':id/repository/compare' do - authorize! :download_code, user_project compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end @@ -105,8 +101,6 @@ module API success Entities::Contributor end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin present user_project.repository.contributors, with: Entities::Contributor diff --git a/lib/api/services.rb b/lib/api/services.rb index bc427705777..d11cdce4e18 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,84 +1,645 @@ module API - # Projects API class Services < Grape::API + services = { + 'asana' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'User API token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' + } + ], + 'assembla' => [ + { + required: true, + name: :token, + type: String, + desc: 'The authentication token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Subdomain setting' + } + ], + 'bamboo' => [ + { + required: true, + name: :bamboo_url, + type: String, + desc: 'Bamboo root URL like https://bamboo.example.com' + }, + { + required: true, + name: :build_key, + type: String, + desc: 'Bamboo build plan key like' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with API access, if applicable' + }, + { + required: true, + name: :password, + type: String, + desc: 'Passord of the user' + } + ], + 'bugzilla' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'buildkite' => [ + { + required: true, + name: :token, + type: String, + desc: 'Buildkite project GitLab token' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The buildkite project URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'builds-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :add_pusher, + type: Boolean, + desc: 'Add pusher to recipients list' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'campfire' => [ + { + required: true, + name: :token, + type: String, + desc: 'Campfire token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Campfire subdomain' + }, + { + required: false, + name: :room, + type: String, + desc: 'Campfire room' + }, + ], + 'custom-issue-tracker' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'drone-ci' => [ + { + required: true, + name: :token, + type: String, + desc: 'Drone CI token' + }, + { + required: true, + name: :drone_url, + type: String, + desc: 'Drone CI URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'emails-on-push' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :disable_diffs, + type: Boolean, + desc: 'Disable code diffs' + }, + { + required: false, + name: :send_from_committer_email, + type: Boolean, + desc: 'Send from committer' + } + ], + 'external-wiki' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'The URL of the external Wiki' + } + ], + 'flowdock' => [ + { + required: true, + name: :token, + type: String, + desc: 'Flowdock token' + } + ], + 'gemnasium' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'Your personal API key on gemnasium.com' + }, + { + required: true, + name: :token, + type: String, + desc: "The project's slug on gemnasium.com" + } + ], + 'hipchat' => [ + { + required: true, + name: :token, + type: String, + desc: 'The room token' + }, + { + required: false, + name: :room, + type: String, + desc: 'The room name or ID' + }, + { + required: false, + name: :color, + type: String, + desc: 'The room color' + }, + { + required: false, + name: :notify, + type: Boolean, + desc: 'Enable notifications' + }, + { + required: false, + name: :api_version, + type: String, + desc: 'Leave blank for default (v2)' + }, + { + required: false, + name: :server, + type: String, + desc: 'Leave blank for default. https://hipchat.example.com' + } + ], + 'irker' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Recipients/channels separated by whitespaces' + }, + { + required: false, + name: :default_irc_uri, + type: String, + desc: 'Default: irc://irc.network.net:6697' + }, + { + required: false, + name: :server_host, + type: String, + desc: 'Server host. Default localhost' + }, + { + required: false, + name: :server_port, + type: Integer, + desc: 'Server port. Default 6659' + }, + { + required: false, + name: :colorize_messages, + type: Boolean, + desc: 'Colorize messages' + } + ], + 'jira' => [ + { + required: true, + name: :url, + type: String, + desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + }, + { + required: true, + name: :project_key, + type: String, + desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :password, + type: String, + desc: 'The password of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :jira_issue_transition_id, + type: Integer, + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + } + ], + + 'kubernetes' => [ + { + required: true, + name: :namespace, + type: String, + desc: 'The Kubernetes namespace to use' + }, + { + required: true, + name: :api_url, + type: String, + desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' + }, + { + required: true, + name: :token, + type: String, + desc: 'The service token to authenticate against the Kubernetes cluster with' + }, + { + required: false, + name: :ca_pem, + type: String, + desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + }, + ], + 'mattermost-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], + 'pipelines-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'pivotaltracker' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Pivotaltracker token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' + } + ], + 'pushover' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'The application key' + }, + { + required: true, + name: :user_key, + type: String, + desc: 'The user key' + }, + { + required: true, + name: :priority, + type: String, + desc: 'The priority' + }, + { + required: true, + name: :device, + type: String, + desc: 'Leave blank for all active devices' + }, + { + required: true, + name: :sound, + type: String, + desc: 'The sound of the notification' + } + ], + 'redmine' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'The new issue URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], + 'slack' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' + }, + { + required: false, + name: :new_issue_url, + type: String, + desc: 'The user name' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The channel name' + } + ], + 'mattermost' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], + 'teamcity' => [ + { + required: true, + name: :teamcity_url, + type: String, + desc: 'TeamCity root URL like https://teamcity.example.com' + }, + { + required: true, + name: :build_type, + type: String, + desc: 'Build configuration ID' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with permissions to trigger a manual build' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user' + } + ] + }.freeze + + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ] + }.freeze + resource :projects do before { authenticate! } before { authorize_admin_project } - # Set <service_slug> service for project - # - # Example Request: - # - # PUT /projects/:id/services/gitlab-ci - # - put ':id/services/:service_slug' do - if project_service - validators = project_service.class.validators.select do |s| - s.class == ActiveRecord::Validations::PresenceValidator && - s.attributes != [:project_id] + helpers do + def service_attributes(service) + service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym end + end + end - required_attributes! validators.map(&:attributes).flatten.uniq - attrs = attributes_for_keys service_attributes + services.each do |service_slug, settings| + desc "Set #{service_slug} service for project" + params do + settings.each do |setting| + if setting[:required] + requires setting[:name], type: setting[:type], desc: setting[:desc] + else + optional setting[:name], type: setting[:type], desc: setting[:desc] + end + end + end + put ":id/services/#{service_slug}" do + service = user_project.find_or_initialize_service(service_slug.underscore) + service_params = declared_params(include_missing: false).merge(active: true) - if project_service.update_attributes(attrs.merge(active: true)) + if service.update_attributes(service_params) true else - not_found! + render_api_error!('400 Bad Request', 400) end end end - # Delete <service_slug> service for project - # - # Example Request: - # - # DELETE /project/:id/services/gitlab-ci - # - delete ':id/services/:service_slug' do - if project_service - attrs = service_attributes.inject({}) do |hash, key| - hash.merge!(key => nil) - end + desc "Delete a service for project" + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + delete ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) - if project_service.update_attributes(attrs.merge(active: false)) - true - else - not_found! - end + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end + + if service.update_attributes(attrs.merge(active: false)) + true + else + render_api_error!('400 Bad Request', 400) end end - # Get <service_slug> service settings for project - # - # Example Request: - # - # GET /project/:id/services/gitlab-ci - # - get ':id/services/:service_slug' do - present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? end end - resource :projects do - desc 'Trigger a slash command' do - detail 'Added in GitLab 8.13' + trigger_services.each do |service_slug, settings| + params do + requires :id, type: String, desc: 'The ID of a project' end - post ':id/services/:service_slug/trigger' do - project = find_project(params[:id]) + resource :projects do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] + end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) - # This is not accurate, but done to prevent leakage of the project names - not_found!('Service') unless project + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project - service = project_service(project) + service = project.find_or_initialize_service(service_slug.underscore) - result = service.try(:active?) && service.try(:trigger, params) + result = service.try(:active?) && service.try(:trigger, params) - if result - status result[:status] || 200 - present result - else - not_found!('Service') + if result + status result[:status] || 200 + present result + else + not_found!('Service') + end end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c4cb1c7924a..9eb9a105bde 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -9,23 +9,117 @@ module API end end - # Get current applicaiton settings - # - # Example Request: - # GET /application/settings + desc 'Get the current application settings' do + success Entities::ApplicationSetting + end get "application/settings" do present current_settings, with: Entities::ApplicationSetting end - # Modify application settings - # - # Example Request: - # PUT /application/settings + desc 'Modify application settings' do + success Entities::ApplicationSetting + end + params do + optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' + optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' + optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' + optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' + given domain_blacklist_enabled: ->(val) { val } do + requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + end + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' + end + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' + end + optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have" + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' + given metrics_enabled: ->(val) { val } do + requires :metrics_host, type: String, desc: 'The InfluxDB host' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' + requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + end + optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' + given sidekiq_throttling_enabled: ->(val) { val } do + requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' + requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + end + optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' + given recaptcha_enabled: ->(val) { val } do + requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' + requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' + end + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' + given sentry_enabled: ->(val) { val } do + requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' + end + optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :koding_enabled, type: Boolean, desc: 'Enable Koding' + given koding_enabled: ->(val) { val } do + requires :koding_url, type: String, desc: 'The Koding team URL' + end + optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + end + at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, + :default_group_visibility, :restricted_visibility_levels, :import_sources, + :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, + :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, + :user_oauth_applications, :user_default_external, :signup_enabled, + :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, + :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, + :shared_runners_enabled, :max_artifacts_size, :container_registry_token_expire_delay, + :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, + :akismet_enabled, :admin_notification_email, :sentry_enabled, + :repository_storage, :repository_checks_enabled, :koding_enabled, + :version_check_enabled, :email_author_in_body, :html_emails_enabled, + :housekeeping_enabled + end put "application/settings" do - attributes = ["repository_storage"] + current_settings.attributes.keys - ["id"] - attrs = attributes_for_keys(attributes) - - if current_settings.update_attributes(attrs) + if current_settings.update_attributes(declared_params(include_missing: false)) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 8a53d9c0095..e23f99256a5 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -8,6 +8,10 @@ module API gitlab_ci_ymls: { klass: Gitlab::Template::GitlabCiYmlTemplate, gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 } }.freeze PROJECT_TEMPLATE_REGEX = @@ -51,7 +55,7 @@ module API end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' - end + end get route do options = { featured: declared(params).popular.present? ? true : nil @@ -69,7 +73,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route, requirements: { name: /[\w\.-]+/ } do not_found!('License') unless Licensee::License.find(declared(params).name) @@ -78,7 +82,7 @@ module API present template, with: Entities::RepoLicense end end - + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| klass = properties[:klass] gitlab_version = properties[:gitlab_version] @@ -104,7 +108,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route do new_template = klass.find(declared(params).name) diff --git a/lib/api/users.rb b/lib/api/users.rb index 1dab799dd61..de07fbf59fc 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -2,7 +2,10 @@ module API class Users < Grape::API include PaginationParams - before { authenticate! } + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do @@ -13,7 +16,7 @@ module API optional :website_url, type: String, desc: 'The website of the user' optional :organization, type: String, desc: 'The organization of the user' optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' - optional :extern_uid, type: Integer, desc: 'The external authentication provider UID' + optional :extern_uid, type: String, desc: 'The external authentication provider UID' optional :provider, type: String, desc: 'The external provider' optional :bio, type: String, desc: 'The biography of the user' optional :location, type: String, desc: 'The location of the user' @@ -91,7 +94,7 @@ module API identity_attrs = params.slice(:provider, :extern_uid) confirm = params.delete(:confirm) - user = User.build_user(declared_params(include_missing: false)) + user = User.new(declared_params(include_missing: false)) user.skip_confirmation! unless confirm if identity_attrs.any? @@ -353,7 +356,7 @@ module API success Entities::UserPublic end get do - present current_user, with: @impersonator ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index d904a8bd4ae..6d04f68c8f9 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -248,21 +248,32 @@ module Banzai end def projects_relation_for_paths(paths) - Project.where_paths_in(paths).includes(:namespace) + Project.where_full_path_in(paths).includes(:namespace) end # Returns projects for the given paths. def find_projects_for_paths(paths) if RequestStore.active? - to_query = paths - project_refs_cache.keys + cache = project_refs_cache + to_query = paths - cache.keys unless to_query.empty? - projects_relation_for_paths(to_query).each do |project| - get_or_set_cache(project_refs_cache, project.path_with_namespace) { project } + projects = projects_relation_for_paths(to_query) + + found = [] + projects.each do |project| + ref = project.path_with_namespace + get_or_set_cache(cache, ref) { project } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } end end - project_refs_cache.slice(*paths).values + cache.slice(*paths).values.compact else projects_relation_for_paths(paths) end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 2f19b59e725..d67d466bce8 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -10,7 +10,7 @@ module Banzai node.set_attribute('href', href) end - if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) node.set_attribute('rel', 'nofollow noreferrer') node.set_attribute('target', '_blank') end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb new file mode 100644 index 00000000000..b6e784c886b --- /dev/null +++ b/lib/banzai/filter/math_filter.rb @@ -0,0 +1,46 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # + class MathFilter < HTML::Pipeline::Filter + # Attribute indicating inline or display math. + STYLE_ATTRIBUTE = 'data-math-style'.freeze + + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-math'.freeze + + INLINE_CLASSES = "code math #{TAG_CLASS}".freeze + + DOLLAR_SIGN = '$'.freeze + + def call + doc.css('code').each do |code| + closing = code.next + opening = code.previous + + # We need a sibling before and after. + # They should end and start with $ respectively. + if closing && opening && + closing.text? && opening.text? && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = INLINE_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + end + end + + doc.css('pre.code.math').each do |el| + el[STYLE_ATTRIBUTE] = 'display' + el[:class] += " #{TAG_CLASS}" + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index f09d78be0ce..9e23c8f8c55 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -46,7 +46,7 @@ module Banzai end def rebuild_relative_uri(uri) - file_path = relative_file_path(uri.path) + file_path = relative_file_path(uri) uri.path = [ relative_url_root, @@ -59,8 +59,10 @@ module Banzai uri end - def relative_file_path(path) - nested_path = build_relative_path(path, context[:requested_path]) + def relative_file_path(uri) + path = Addressable::URI.unescape(uri.path) + request_path = Addressable::URI.unescape(context[:requested_path]) + nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path end @@ -108,11 +110,7 @@ module Banzai end def uri_type(path) - @uri_types[path] ||= begin - unescaped_path = Addressable::URI.unescape(path) - - current_commit.uri_type(unescaped_path) - end + @uri_types[path] ||= current_commit.uri_type(path) end def current_commit diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5da2d0b008c..5a1f873496c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -6,6 +6,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, + Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..7e55cf4deab --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..423eff8f2a5 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,71 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def has_wiki? + raw['has_wiki'] + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index ed87a2603e8..142bce82286 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -41,7 +41,7 @@ module Ci put ":id" do authenticate_runner! build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - forbidden!('Build has been erased!') if build.erased? + validate_build!(build) update_runner_info @@ -71,9 +71,7 @@ module Ci # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) - forbidden!('Build has been erased!') if build.erased? + authenticate_build!(build) error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] @@ -104,8 +102,7 @@ module Ci Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('build is not running') unless build.running? if params[:filesize] @@ -142,10 +139,8 @@ module Ci require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('Build is not running!') unless build.running? - forbidden!('Build has been erased!') if build.erased? artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) @@ -176,8 +171,7 @@ module Ci # GET /builds/:id/artifacts get ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -202,8 +196,7 @@ module Ci # DELETE /builds/:id/artifacts delete ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) build.erase_artifacts! end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e608f5f6cad..5ff25a3a9b2 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,8 +13,19 @@ module Ci forbidden! unless current_runner end - def authenticate_build_token!(build) - forbidden! unless build_token_valid?(build) + def authenticate_build!(build) + validate_build!(build) do + forbidden! unless build_token_valid?(build) + end + end + + def validate_build!(build) + not_found! unless build + + yield if block_given? + + forbidden!('Project has been deleted!') unless build.project + forbidden!('Build has been erased!') if build.erased? end def runner_registration_token_valid? @@ -49,7 +60,7 @@ module Ci end def build_not_found! - if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + if headers['User-Agent'].to_s.match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) no_content! else not_found! diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index fef652cb975..7463bd719d5 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -118,7 +118,7 @@ module Ci .merge(job_variables(name)) variables.map do |key, value| - { key: key, value: value, public: true } + { key: key.to_s, value: value, public: true } end end diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb new file mode 100644 index 00000000000..f48abcc86d5 --- /dev/null +++ b/lib/gitlab/allowable.rb @@ -0,0 +1,7 @@ +module Gitlab + module Allowable + def can?(user, action, subject) + Ability.allowed?(user, action, subject) + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 9667df4ffb8..fa234284361 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,4 +1,5 @@ require 'asciidoctor' +require 'asciidoctor/converter/html5' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -23,7 +24,7 @@ module Gitlab def self.render(input, context, asciidoc_opts = {}) asciidoc_opts.reverse_merge!( safe: :secure, - backend: :html5, + backend: :gitlab_html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) @@ -34,5 +35,29 @@ module Gitlab html.html_safe end + + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index aca5d0020cf..8dda65c71ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,10 @@ module Gitlab module Auth class MissingPersonalTokenError < StandardError; end + SCOPES = [:api, :read_user] + DEFAULT_SCOPES = [:api] + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -88,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -97,12 +101,27 @@ module Gitlab def personal_access_token_check(login, password) if login && password - user = User.find_by_personal_access_token(password) + token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + + if valid_personal_access_token?(token, validation) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end end end + def valid_oauth_token?(token) + token && token.accessible? && valid_api_token?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && valid_api_token?(token) + end + + def valid_api_token?(token) + AccessTokenValidationService.new(token).include_any_scope?(['api']) + end + def lfs_token_check(login, password) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 6be7f690676..39b86c61a18 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -9,8 +9,7 @@ module Gitlab def lfs_deploy_token?(for_project) type == :lfs_deploy_token && - actor && - actor.projects.include?(for_project) + actor.try(:has_access_to?, for_project) end def success? diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb index 50aa45e5406..b762d85b6e5 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/build/status.rb @@ -20,8 +20,8 @@ module Gitlab def status @project.pipelines - .where(sha: @sha, ref: @ref) - .status || 'unknown' + .where(sha: @sha) + .latest_status(@ref) || 'unknown' end def metadata diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..44323b47dca 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,247 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + include Gitlab::ShellAdapter + + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_wiki + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def identifier - project.import_source + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) end - def has_issues? - client.project(identifier)["has_issues"] + def repo + @repo ||= client.repo(project.import_source) end - def import_issues - issues = client.issues(identifier) + def import_wiki + return if project.wiki.repository_exists? - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + path_with_namespace = "#{project.path_with_namespace}.wiki" + import_url = project.import_url.sub(/\.git\z/, ".git/wiki") + gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + rescue StandardError => e + errors << { type: :wiki, errors: e.message } + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_issues + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - comments = client.issue_comments(identifier, issue["local_id"]) + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if comments.any? - body += @formatter.comments_header + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end + + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..d94f70fd1fb 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,17 +14,24 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data }, + skip_wiki: skip_wiki ).execute end + + private + + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index 25da8474e95..4fe53ce93a9 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -42,6 +42,10 @@ module Gitlab def find_by_iid(iid) collection.find_by(iid: iid) end + + def presenter + Gitlab::ChatCommands::Presenter.new + end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index b0d3fdbc48a..145086755e4 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -22,8 +22,6 @@ module Gitlab end end - private - def match_command match = nil service = available_commands.find do |klass| @@ -33,6 +31,8 @@ module Gitlab [service, match] end + private + def help_messages available_commands.map(&:help_message) end @@ -48,15 +48,15 @@ module Gitlab end def help(messages) - Mattermost::Presenter.help(messages, params[:command]) + presenter.help(messages, params[:command]) end def access_denied - Mattermost::Presenter.access_denied + presenter.access_denied end def present(resource) - Mattermost::Presenter.present(resource) + presenter.present(resource) end end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb index 0eed1fce0dc..7127d2f6d04 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/chat_commands/deploy.rb @@ -4,7 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers def self.match(text) - /\Adeploy\s+(?<from>.*)\s+to+\s+(?<to>.*)\z/.match(text) + /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) end def self.help_message @@ -50,7 +50,8 @@ module Gitlab def url(subject) polymorphic_url( - [ subject.project.namespace.becomes(Namespace), subject.project, subject ]) + [subject.project.namespace.becomes(Namespace), subject.project, subject] + ) end end end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb index 1dba85c1b51..cefb6775db8 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -8,7 +8,7 @@ module Gitlab end def self.help_message - 'issue new <title>\n<description>' + 'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>' end def self.allowed?(project, user) diff --git a/lib/mattermost/presenter.rb b/lib/gitlab/chat_commands/presenter.rb index 67eda983a74..8930a21f406 100644 --- a/lib/mattermost/presenter.rb +++ b/lib/gitlab/chat_commands/presenter.rb @@ -1,7 +1,7 @@ -module Mattermost - class Presenter - class << self - include Gitlab::Routing.url_helpers +module Gitlab + module ChatCommands + class Presenter + include Gitlab::Routing def authorize_chat_name(url) message = if url @@ -30,12 +30,12 @@ module Mattermost if subject.is_a?(Gitlab::ChatCommands::Result) show_result(subject) elsif subject.respond_to?(:count) - if subject.many? - multiple_resources(subject) - elsif subject.none? + if subject.none? not_found + elsif subject.one? + single_resource(subject.first) else - single_resource(subject) + multiple_resources(subject) end else single_resource(subject) @@ -64,16 +64,16 @@ module Mattermost def single_resource(resource) return error(resource) if resource.errors.any? || !resource.persisted? - message = "### #{title(resource)}" + message = "#{title(resource)}:" message << "\n\n#{resource.description}" if resource.try(:description) in_channel_response(message) end def multiple_resources(resources) - resources.map! { |resource| title(resource) } + titles = resources.map { |resource| title(resource) } - message = header_with_list("Multiple results were found:", resources) + message = header_with_list("Multiple results were found:", titles) ephemeral_response(message) end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index cb1065223d4..9c391fa92a3 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,13 +1,16 @@ module Gitlab module Checks class ChangeAccess - attr_reader :user_access, :project + attr_reader :user_access, :project, :skip_authorization - def initialize(change, user_access:, project:) + def initialize( + change, user_access:, project:, env: {}, skip_authorization: false) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project + @env = env + @skip_authorization = skip_authorization end def exec @@ -23,6 +26,7 @@ module Gitlab protected def protected_branch_checks + return if skip_authorization return unless @branch_name return unless project.protected_branch?(@branch_name) @@ -48,6 +52,8 @@ module Gitlab end def tag_checks + return if skip_authorization + tag_ref = Gitlab::Git.tag_name(@ref) if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) @@ -56,6 +62,8 @@ module Gitlab end def push_checks + return if skip_authorization + if user_access.cannot_do_action?(:push_code) "You are not allowed to push code to this project." end @@ -68,7 +76,7 @@ module Gitlab end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 5fe86553bd0..de0c9049ebf 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,15 +1,20 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev) + def self.force_push?(project, oldrev, newrev, env: {}) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) - missed_ref.present? + missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute + + if exit_status == 0 + missed_ref.present? + else + raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." + end end end end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb new file mode 100644 index 00000000000..a979fe7d573 --- /dev/null +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Cancelable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'ban' + end + + def action_path + cancel_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def action_title + 'Cancel' + end + + def self.matches?(build, user) + build.cancelable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb new file mode 100644 index 00000000000..3fec2c5d4db --- /dev/null +++ b/lib/gitlab/ci/status/build/common.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + module Build + module Common + def has_details? + can?(user, :read_build, subject) + end + + def details_path + namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb new file mode 100644 index 00000000000..eee9a64120b --- /dev/null +++ b/lib/gitlab/ci/status/build/factory.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + module Status + module Build + class Factory < Status::Factory + def self.extended_statuses + [Status::Build::Stop, Status::Build::Play, + Status::Build::Cancelable, Status::Build::Retryable] + end + + def self.common_helpers + Status::Build::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb new file mode 100644 index 00000000000..1bf949c96dd --- /dev/null +++ b/lib/gitlab/ci/status/build/play.rb @@ -0,0 +1,57 @@ +module Gitlab + module Ci + module Status + module Build + class Play < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual play action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'play' + end + + def action_title + 'Play' + end + + def action_class + 'ci-play-icon' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && !build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb new file mode 100644 index 00000000000..8e38d6a8523 --- /dev/null +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Retryable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'refresh' + end + + def action_title + 'Retry' + end + + def action_path + retry_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.retryable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb new file mode 100644 index 00000000000..e1dfdb76d41 --- /dev/null +++ b/lib/gitlab/ci/status/build/stop.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + module Status + module Build + class Stop < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual stop action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'stop' + end + + def action_title + 'Stop' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index ce4108fdcf2..73b6ab5a635 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -4,10 +4,14 @@ module Gitlab # Base abstract class fore core status # class Core - include Gitlab::Routing.url_helpers + include Gitlab::Routing + include Gitlab::Allowable - def initialize(subject) + attr_reader :subject, :user + + def initialize(subject, user) @subject = subject + @user = user end def icon @@ -18,23 +22,12 @@ module Gitlab raise NotImplementedError end - def title - "#{@subject.class.name.demodulize}: #{label}" - end - - # Deprecation warning: this method is here because we need to maintain - # backwards compatibility with legacy statuses. We often do something - # like "ci-status ci-status-#{status}" to set CSS class. - # - # `to_s` method should be renamed to `group` at some point, after - # phasing legacy satuses out. - # - def to_s - self.class.name.demodulize.downcase.underscore + def group + self.class.name.demodulize.underscore end def has_details? - raise NotImplementedError + false end def details_path @@ -42,16 +35,27 @@ module Gitlab end def has_action? - raise NotImplementedError + false end def action_icon raise NotImplementedError end + def action_class + end + def action_path raise NotImplementedError end + + def action_method + raise NotImplementedError + end + + def action_title + raise NotImplementedError + end end end end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index 6bfb5d38c1f..d367c9bda69 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -2,8 +2,12 @@ module Gitlab module Ci module Status module Extended - def matches?(_subject) - raise NotImplementedError + extend ActiveSupport::Concern + + class_methods do + def matches?(_subject, _user) + raise NotImplementedError + end end end end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index b2f896f2211..ae9ef895df4 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -2,10 +2,9 @@ module Gitlab module Ci module Status class Factory - attr_reader :subject - - def initialize(subject) + def initialize(subject, user) @subject = subject + @user = user end def fabricate! @@ -16,27 +15,32 @@ module Gitlab end end + def self.extended_statuses + [] + end + + def self.common_helpers + Module.new + end + private - def subject_status - @subject_status ||= subject.status + def simple_status + @simple_status ||= @subject.status || :created end def core_status Gitlab::Ci::Status - .const_get(subject_status.capitalize) - .new(subject) + .const_get(simple_status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) end def extended_status - @extended ||= extended_statuses.find do |status| - status.matches?(subject) + @extended ||= self.class.extended_statuses.find do |status| + status.matches?(@subject, @user) end end - - def extended_statuses - [] - end end end end diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 25e52bec3da..76bfd18bf40 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -4,13 +4,13 @@ module Gitlab module Pipeline module Common def has_details? - true + can?(user, :read_pipeline, subject) end def details_path - namespace_project_pipeline_path(@subject.project.namespace, - @subject.project, - @subject) + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject) end def has_action? diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 4ac4ec671d0..16dcb326be9 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -3,14 +3,12 @@ module Gitlab module Status module Pipeline class Factory < Status::Factory - private - - def extended_statuses + def self.extended_statuses [Pipeline::SuccessWithWarnings] end - def core_status - super.extend(Status::Pipeline::Common) + def self.common_helpers + Status::Pipeline::Common end end end diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb index 4b040d60df8..24bf8b869e0 100644 --- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb +++ b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb @@ -3,7 +3,7 @@ module Gitlab module Status module Pipeline class SuccessWithWarnings < SimpleDelegator - extend Status::Extended + include Status::Extended def text 'passed' @@ -17,11 +17,11 @@ module Gitlab 'icon_status_warning' end - def to_s + def group 'success_with_warnings' end - def self.matches?(pipeline) + def self.matches?(pipeline, user) pipeline.success? && pipeline.has_warnings? end end diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index 14c437d2b98..7852f492e1d 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -4,14 +4,14 @@ module Gitlab module Stage module Common def has_details? - true + can?(user, :read_pipeline, subject.pipeline) end def details_path - namespace_project_pipeline_path(@subject.project.namespace, - @subject.project, - @subject.pipeline, - anchor: @subject.name) + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject.pipeline, + anchor: subject.name) end def has_action? diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index c6522d5ada1..689a5dd45bc 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -3,10 +3,8 @@ module Gitlab module Status module Stage class Factory < Status::Factory - private - - def core_status - super.extend(Status::Stage::Common) + def self.common_helpers + Status::Stage::Common end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index c6bb8f9c8ed..9d142f1b82e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 56530448f36..329d12f13d1 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -61,7 +61,10 @@ module Gitlab end def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.blob && diff_file.blob.text? + @merge_request_diff.present? && + diff_file.blob && + diff_file.blob.text? && + @project.repository.diffable?(diff_file.blob) end def cache_key diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 85402c2a278..8c8dd1b9cef 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -13,9 +13,17 @@ module Gitlab encoding = body.encoding - body = discourse_email_trimmer(body) + body = EmailReplyTrimmer.trim(body) - body = EmailReplyParser.parse_reply(body) + return '' unless body + + # not using /\s+$/ here because that deletes empty lines + body = body.gsub(/[ \t]$/, '') + + # NOTE: We currently don't support empty quotes. + # EmailReplyTrimmer allows this as a special case, + # so we detect it manually here. + return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } body.force_encoding(encoding).encode("UTF-8") end @@ -57,30 +65,6 @@ module Gitlab rescue nil end - - REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) - REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) - - def discourse_email_trimmer(body) - lines = body.scrub.lines.to_a - range_end = 0 - - lines.each_with_index do |l, idx| - # This one might be controversial but so many reply lines have years, times and end with a colon. - # Let's try it and see how well it works. - break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) - - # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } - # Headers on the same line - break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 - - range_end = idx - end - - lines[0..range_end].join.strip - end end end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index a7c596dced0..b984492d369 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -76,7 +76,7 @@ module Gitlab if referable.respond_to?(:project) referable.to_reference(target_project) else - referable.to_reference(@source_project, target_project) + referable.to_reference(@source_project, target_project: target_project) end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb new file mode 100644 index 00000000000..79dd0cf7df2 --- /dev/null +++ b/lib/gitlab/git/rev_list.rb @@ -0,0 +1,42 @@ +module Gitlab + module Git + class RevList + attr_reader :project, :env + + ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze + + def initialize(oldrev, newrev, project:, env: nil) + @project = project + @env = env.presence || {} + @args = [Gitlab.config.git.bin_path, + "--git-dir=#{project.repository.path_to_repo}", + "rev-list", + "--max-count=1", + oldrev, + "^#{newrev}"] + end + + def execute + Gitlab::Popen.popen(@args, nil, parse_environment_variables) + end + + def valid? + environment_variables.all? do |(name, value)| + value.to_s.start_with?(project.repository.path_to_repo) + end + end + + private + + def parse_environment_variables + return {} unless valid? + + environment_variables + end + + def environment_variables + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db07b7c5fcc..7e1484613f2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -7,7 +7,8 @@ module Gitlab ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', - deploy_key: 'Deploy keys are not allowed to push code.', + deploy_key_upload: + 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.' } @@ -17,12 +18,13 @@ module Gitlab attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, env: {}) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) + @env = env end def check(cmd, changes) @@ -30,12 +32,13 @@ module Gitlab check_active_user! check_project_accessibility! check_command_existence!(cmd) + check_repository_existence! case cmd when *DOWNLOAD_COMMANDS - download_access_check + check_download_access! when *PUSH_COMMANDS - push_access_check(changes) + check_push_access!(changes) end build_status_object(true) @@ -43,32 +46,10 @@ module Gitlab build_status_object(false, ex.message) end - def download_access_check - if user - user_download_access_check - elsif deploy_key.nil? && !guest_can_downlod_code? - raise UnauthorizedError, ERROR_MESSAGES[:download] - end - end - - def push_access_check(changes) - if user - user_push_access_check(changes) - else - raise UnauthorizedError, ERROR_MESSAGES[deploy_key ? :deploy_key : :upload] - end - end - - def guest_can_downlod_code? + def guest_can_download_code? Guest.can?(:download_code, project) end - def user_download_access_check - unless user_can_download_code? || build_can_download_code? - raise UnauthorizedError, ERROR_MESSAGES[:download] - end - end - def user_can_download_code? authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) end @@ -77,35 +58,6 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end - def user_push_access_check(changes) - unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:upload] - end - - if changes.blank? - return # Allow access. - end - - unless project.repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] - end - - changes_list = Gitlab::ChangesList.new(changes) - - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each do |change| - status = change_access_check(change) - unless status.allowed? - # If user does not have access to make at least one change - cancel all push - raise UnauthorizedError, status.message - end - end - end - - def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec - end - def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -119,6 +71,8 @@ module Gitlab end def check_active_user! + return if deploy_key? + if user && !user_access.allowed? raise UnauthorizedError, "Your account has been blocked." end @@ -136,33 +90,92 @@ module Gitlab end end - def matching_merge_request?(newrev, branch_name) - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + def check_repository_existence! + unless project.repository.exists? + raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + end end - def deploy_key - actor if actor.is_a?(DeployKey) + def check_download_access! + return if deploy_key? + + passed = user_can_download_code? || + build_can_download_code? || + guest_can_download_code? + + unless passed + raise UnauthorizedError, ERROR_MESSAGES[:download] + end end - def deploy_key_can_read_project? + def check_push_access!(changes) if deploy_key - return true if project.public? - deploy_key.projects.include?(project) + check_deploy_key_push_access! + elsif user + check_user_push_access! else - false + raise UnauthorizedError, ERROR_MESSAGES[:upload] end + + return if changes.blank? # Allow access. + + check_change_access!(changes) end - def can_read_project? - if user - user_access.can_read_project? - elsif deploy_key - deploy_key_can_read_project? - else - Guest.can?(:read_project, project) + def check_user_push_access! + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:upload] end end + def check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + end + end + + def check_change_access!(changes) + changes_list = Gitlab::ChangesList.new(changes) + + # Iterate over all changes to find if user allowed all of them to be applied + changes_list.each do |change| + status = check_single_change_access(change) + unless status.allowed? + # If user does not have access to make at least one change - cancel all push + raise UnauthorizedError, status.message + end + end + end + + def check_single_change_access(change) + Checks::ChangeAccess.new( + change, + user_access: user_access, + project: project, + env: @env, + skip_authorization: deploy_key?).exec + end + + def matching_merge_request?(newrev, branch_name) + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + end + + def deploy_key + actor if deploy_key? + end + + def deploy_key? + actor.is_a?(DeployKey) + end + + def can_read_project? + if deploy_key + deploy_key.has_access_to?(project) + elsif user + user.can?(:read_project, project) + end || Guest.can?(:read_project, project) + end + protected def user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 2c06c4ff1ef..67eaa5e088d 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,6 @@ module Gitlab class GitAccessWiki < GitAccess - def guest_can_downlod_code? + def guest_can_download_code? Guest.can?(:download_wiki_code, project) end @@ -8,7 +8,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def change_access_check(change) + def check_single_change_access(change) if user_access.can_do_action?(:create_wiki) build_status_object(true) else diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 6dbae64a9fe..95dba9a327b 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,6 +15,10 @@ module Gitlab end end + def url + raw_data.url || '' + end + private def gitlab_user_id(github_id) diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 85df6547a67..ba869faa92e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,10 +4,12 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version if access_token ::Octokit.auto_paginate = false @@ -17,7 +19,7 @@ module Gitlab def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -64,6 +66,14 @@ module Gitlab private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 281b65bdeba..ec1318ab33c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,7 +3,7 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :errors, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @@ -11,12 +11,27 @@ module Gitlab @repo_url = project.import_url @errors = [] @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute @@ -35,7 +50,13 @@ module Gitlab import_comments(:issues) import_comments(:pull_requests) import_wiki - import_releases + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + handle_errors true @@ -44,7 +65,9 @@ module Gitlab private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil end def handle_errors @@ -60,9 +83,10 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - LabelFormatter.new(project, raw).create! + gh_label = LabelFormatter.new(project, raw) + gh_label.create! rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end end @@ -74,9 +98,10 @@ module Gitlab fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - MilestoneFormatter.new(project, raw).create! + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end end @@ -97,7 +122,7 @@ module Gitlab apply_labels(issuable, raw) rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end end @@ -106,18 +131,23 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = PullRequestFormatter.new(project, raw) - next unless pull_request.valid? + gh_pull_request = PullRequestFormatter.new(project, raw) + next unless gh_pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! - pull_request.create! + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure - clean_up_restored_branches(pull_request) + clean_up_restored_branches(gh_pull_request) end end end @@ -233,7 +263,7 @@ module Gitlab gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..256f360efc7 --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,60 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + def project_association + raise NotImplementedError + end + + def number + raw_data.number + end + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gitlab_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gitlab_author_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + if gitlab_author_id + body + else + formatter.author_line(author) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 887690bcc7c..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -24,59 +24,9 @@ module Gitlab :issues end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def pull_request? raw_data.pull_request.present? end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body - else - formatter.author_line(author) + body - end - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end end end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 401dd962521..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.number, + iid: number, project: project, title: raw_data.title, description: raw_data.description, @@ -19,7 +19,15 @@ module Gitlab end def find_condition - { iid: raw_data.number } + { iid: number } + end + + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index a2410068845..3f635be22ba 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,14 +1,15 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :name, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute @@ -19,7 +20,7 @@ module Gitlab description: repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - import_type: "github", + import_type: type, import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki @@ -29,7 +30,7 @@ module Gitlab private def import_url - repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b9a227fb11a..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -28,14 +28,6 @@ module Gitlab :merge_requests end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def valid? source_branch.valid? && target_branch.valid? end @@ -60,57 +52,15 @@ module Gitlab end end - def url - raw_data.url - end - private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - formatter.author_line(author) + body + super end end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2c21804fe7a..4d4e04e9e35 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,6 +8,8 @@ module Gitlab gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path + gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') + gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c551321c18d..cda6ddf0443 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -120,7 +120,7 @@ module Gitlab members_mapper: members_mapper, user: @user, project_id: restored_project.id) - end + end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index a0e80fccad9..7a649f28340 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -22,7 +22,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze def self.create(*args) new(*args).create @@ -40,6 +40,8 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return nil if unknown_service? + setup_models generate_imported_object @@ -99,6 +101,8 @@ module Gitlab def generate_imported_object if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') + @relation_hash.delete('token') + imported_object do |object| object.trace = trace object.commit_id = nil @@ -185,7 +189,7 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - attribute_hash = attribute_hash_for(['events', 'priorities']) + attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? @@ -206,15 +210,38 @@ module Gitlab def existing_object @existing_object ||= begin - finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] - finder_hash = parsed_relation_hash.slice(*finder_attributes) - existing_object = relation_class.find_or_create_by(finder_hash) + existing_object = find_or_create_object! + # Done in two steps, as MySQL behaves differently than PostgreSQL using # the +find_or_create_by+ method and does not return the ID the second time. existing_object.update!(parsed_relation_hash) existing_object end end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def find_or_create_object! + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) + + if label? + label = relation_class.find_or_initialize_by(finder_hash) + parsed_relation_hash.delete('priorities') if label.persisted? + + label.save! + label + else + relation_class.find_or_create_by(finder_hash) + end + end + + def label? + @relation_name.to_s.include?('label') + end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 94261b7eeed..45958710c13 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,21 +7,38 @@ module Gitlab module ImportSources extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze + class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..288771c1c12 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token, ca_pem = nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:ca_pem] = ca_pem if ca_pem.present? + terminal + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b81f3e8e8f5..333f170a484 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -28,7 +28,7 @@ module Gitlab end def name - entry.cn.first + attribute_value(:name) end def uid @@ -40,7 +40,7 @@ module Gitlab end def email - entry.try(:mail) + attribute_value(:email) end def dn @@ -56,6 +56,21 @@ module Gitlab def config @config ||= Gitlab::LDAP::Config.new(provider) end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_sym]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + # Some LDAP attributes return an array, + # even if it is a single value (like 'cn') + Array(entry.public_send(selected_attr)).first + end end end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 01c96a6fe96..91fb0bb317a 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -70,8 +70,8 @@ module Gitlab def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] - trans.action = "Grape##{endpoint.route.route_method} #{path}" + path = endpoint_paths_cache[endpoint.route.request_method][endpoint.route.path] + trans.action = "Grape##{endpoint.route.request_method} #{path}" end private diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..dd99f9bb7d7 --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,103 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = open_file(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = open_file(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + + def open_file(path) + ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index cc74bb29087..4bc5cda8cb5 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,13 +5,13 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil) + def popen(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end path ||= Dir.pwd - vars = { "PWD" => path } + vars['PWD'] = path options = { chdir: path } unless File.directory?(path) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 66e6b29e798..6bdf3db9cb8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -110,7 +110,7 @@ module Gitlab end def notes - @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a06cf6a989c..9e0b0e5ea98 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,7 +61,7 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@]*\z/.freeze end def file_name_regex_message @@ -69,7 +69,7 @@ module Gitlab end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze + @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze end def file_path_regex_message @@ -123,5 +123,22 @@ module Gitlab def environment_name_regex_message "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end + + def kubernetes_namespace_regex + /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ + end + + def kubernetes_namespace_regex_message + "can contain only letters, digits or '-', and cannot start or end with '-'" + end + + def environment_slug_regex + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + end + + def environment_slug_regex_message + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" + end end end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 5132177de51..632e2d87500 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -1,5 +1,11 @@ module Gitlab module Routing + extend ActiveSupport::Concern + + included do + include Gitlab::Routing.url_helpers + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serialize/ci/variables.rb new file mode 100644 index 00000000000..3a9443bfcd9 --- /dev/null +++ b/lib/gitlab/serialize/ci/variables.rb @@ -0,0 +1,27 @@ +module Gitlab + module Serialize + module Ci + # This serializer could make sure our YAML variables' keys and values + # are always strings. This is more for legacy build data because + # from now on we convert them into strings before saving to database. + module Variables + extend self + + def load(string) + return unless string + + object = YAML.safe_load(string, [Symbol]) + + object.map do |variable| + variable[:key] = variable[:key].to_s + variable + end + end + + def dump(object) + YAML.dump(object) + end + end + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1cd89b3a9c4..222021e8802 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -22,9 +22,7 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map do |rel| - rel.reorder(nil).to_sql - end + @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end fragments.join("\nUNION\n") diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb new file mode 100644 index 00000000000..d5d3e045a42 --- /dev/null +++ b/lib/gitlab/template/dockerfile_template.rb @@ -0,0 +1,30 @@ +module Gitlab + module Template + class DockerfileTemplate < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + 'Dockerfile' + end + + def categories + { + "General" => '' + } + end + + def base_dir + Rails.root.join('vendor/dockerfile') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 8d1a1ed54c9..9d2ecee9756 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -13,8 +13,9 @@ module Gitlab def categories { - "General" => '', - "Pages" => 'Pages' + 'General' => '', + 'Pages' => 'Pages', + 'Auto deploy' => 'autodeploy' } end @@ -25,6 +26,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end + + def dropdown_names(context) + categories = context == 'autodeploy' ? ['Auto deploy'] : ['General', 'Pages'] + super().slice(*categories) + end end end end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index d4020af76f9..19ab76ae80f 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -15,7 +15,7 @@ module Gitlab Theme.new(1, 'Graphite', 'ui_graphite'), Theme.new(2, 'Charcoal', 'ui_charcoal'), Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Gray', 'ui_gray'), + Theme.new(4, 'Black', 'ui_black'), Theme.new(5, 'Violet', 'ui_violet'), Theme.new(6, 'Blue', 'ui_blue') ].freeze diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb new file mode 100644 index 00000000000..ce14cc887d0 --- /dev/null +++ b/lib/gitlab/update_path_error.rb @@ -0,0 +1,3 @@ +module Gitlab + class UpdatePathError < StandardError; end +end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 9858d2e7d83..6c7e673fb9f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -8,6 +8,8 @@ module Gitlab end def can_do_action?(action) + return false if no_user_or_blocked? + @permission_cache ||= {} @permission_cache[action] ||= user.can?(action, project) end @@ -17,7 +19,7 @@ module Gitlab end def allowed? - return false if user.blank? || user.blocked? + return false if no_user_or_blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -27,7 +29,7 @@ module Gitlab end def can_push_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) @@ -40,7 +42,7 @@ module Gitlab end def can_merge_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten @@ -51,9 +53,15 @@ module Gitlab end def can_read_project? - return false unless user + return false if no_user_or_blocked? user.can?(:read_project, project) end + + private + + def no_user_or_blocked? + user.nil? || user.blocked? + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 594439a5d4b..d28bb583fe7 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -95,6 +95,19 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers] + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' @@ -117,8 +130,12 @@ module Gitlab end def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) JWT.decode( - request_headers[INTERNAL_API_REQUEST_HEADER], + encoded_message, secret, true, { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb new file mode 100644 index 00000000000..ec2903b7ec6 --- /dev/null +++ b/lib/mattermost/client.rb @@ -0,0 +1,41 @@ +module Mattermost + class ClientError < Mattermost::Error; end + + class Client + attr_reader :user + + def initialize(user) + @user = user + end + + private + + def with_session(&blk) + Mattermost::Session.new(user).with_session(&blk) + end + + def json_get(path, options = {}) + with_session do |session| + json_response session.get(path, options) + end + end + + def json_post(path, options = {}) + with_session do |session| + json_response session.post(path, options) + end + end + + def json_response(response) + json_response = JSON.parse(response.body) + + unless response.success? + raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error') + end + + json_response + rescue JSON::JSONError + raise Mattermost::ClientError.new('Cannot parse response') + end + end +end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb new file mode 100644 index 00000000000..d1e4bb0eccf --- /dev/null +++ b/lib/mattermost/command.rb @@ -0,0 +1,10 @@ +module Mattermost + class Command < Client + def create(params) + response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + body: params.to_json) + + response['token'] + end + end +end diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb new file mode 100644 index 00000000000..014df175be0 --- /dev/null +++ b/lib/mattermost/error.rb @@ -0,0 +1,3 @@ +module Mattermost + class Error < StandardError; end +end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..377cb7b1021 --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,160 @@ +module Mattermost + class NoSessionError < Mattermost::Error + def message + 'No session could be set up, is Mattermost configured with Single Sign On?' + end + end + + class ConnectionError < Mattermost::Error; end + + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + LEASE_TIMEOUT = 60 + + base_uri Settings.mattermost.host + + attr_accessor :current_resource_owner, :token + + def initialize(current_user) + @current_resource_owner = current_user + end + + def with_session + with_lease do + raise Mattermost::NoSessionError unless create + + begin + yield self + rescue Errno::ECONNREFUSED + raise Mattermost::NoSessionError + ensure + destroy + end + end + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + handle_exceptions do + self.class.get(path, options.merge(headers: @headers)) + end + end + + def post(path, options = {}) + handle_exceptions do + self.class.post(path, options.merge(headers: @headers)) + end + end + + private + + def create + return unless oauth_uri + return unless token_uri + + @token = request_token + @headers = { + Authorization: "Bearer #{@token}" + } + + @token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri = URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= + if oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + response = get(token_uri, follow_redirects: false) + + if 200 <= response.code && response.code < 400 + response.headers['token'] + end + end + + def with_lease + lease_uuid = lease_try_obtain + raise NoSessionError unless lease_uuid + + begin + yield + ensure + Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) + end + end + + def lease_key + "mattermost:session" + end + + def lease_try_obtain + lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) + lease.try_obtain + end + + def handle_exceptions + yield + rescue HTTParty::Error => e + raise Mattermost::ConnectionError.new(e.message) + rescue Errno::ECONNREFUSED + raise Mattermost::ConnectionError.new(e.message) + end + end +end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb new file mode 100644 index 00000000000..784eca6ab5a --- /dev/null +++ b/lib/mattermost/team.rb @@ -0,0 +1,7 @@ +module Mattermost + class Team < Client + def all + json_get('/api/v3/teams/all') + end + end +end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb new file mode 100644 index 00000000000..80784adfd76 --- /dev/null +++ b/lib/rouge/lexers/math.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Math < Lexer + title "A passthrough lexer used for LaTeX input" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..2f7c34a3f31 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -20,6 +20,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab { + default upgrade; + '' close; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -53,6 +58,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab; proxy_pass http://gitlab-workhorse; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..5661394058d 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -24,6 +24,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab_ssl { + default upgrade; + '' close; +} + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -98,6 +103,9 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab_ssl; + proxy_pass http://gitlab-workhorse; } diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index dbdd4e977e8..a2eca74a3c8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -63,8 +63,7 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count + ProjectCacheWorker.perform(project.id) else puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " Errors: #{project.errors.messages}".color(:red) diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake new file mode 100644 index 00000000000..c66a2a263dc --- /dev/null +++ b/lib/tasks/gitlab/ldap.rake @@ -0,0 +1,40 @@ +namespace :gitlab do + namespace :ldap do + desc 'GitLab | LDAP | Rename provider' + task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + old_provider = args[:old_provider] || + prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) + new_provider = args[:new_provider] || + prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue)) + puts '' # Add some separation in the output + + identities = Identity.where(provider: old_provider) + identity_count = identities.count + + if identities.empty? + puts "Found no user identities with '#{old_provider}' provider." + puts 'Please check the provider name and try again.' + exit 1 + end + + plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user') + + unless ENV['force'] == 'yes' + puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'" + puts 'If the new provider is incorrect, users will be unable to sign in' + ask_to_continue + puts '' + end + + updated_count = identities.update_all(provider: new_provider) + + if updated_count == identity_count + puts 'User identities were successfully updated'.color(:green) + else + plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user') + puts 'Some user identities could not be updated'.color(:red) + puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total" + end + end + end +end diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake deleted file mode 100644 index 3bd10b0208b..00000000000 --- a/lib/tasks/gitlab/update_commit_count.rake +++ /dev/null @@ -1,20 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update commit count for projects" - task update_commit_count: :environment do - projects = Project.where(commit_count: 0) - puts "#{projects.size} projects need to be updated. This might take a while." - ask_to_continue unless ENV['force'] == 'yes' - - projects.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.color(:yellow)} ... " - - unless project.repo_exists? - puts "skipping, because the repo is empty".color(:magenta) - next - end - - project.update_commit_count - puts project.commit_count.to_s.color(:green) - end - end -end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 141a0b74ec0..f5caca3ddbf 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,8 +1,12 @@ +require Rails.root.join('lib/gitlab/database') +require Rails.root.join('lib/gitlab/database/migration_helpers') require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') +require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up + AddLowerPathIndexToRoutes.new.up end |