diff options
Diffstat (limited to 'lib')
268 files changed, 5666 insertions, 1389 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 1bf20f76ad6..52cd7cbe3db 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,6 +6,7 @@ module API version 'v3', using: :path do helpers ::API::V3::Helpers + helpers ::API::Helpers::CommonHelpers mount ::API::V3::AwardEmoji mount ::API::V3::Boards @@ -44,6 +45,9 @@ module API end before { allow_access_with_scope :api } + before { Gitlab::I18n.set_locale(current_user) } + + after { Gitlab::I18n.reset_locale } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -77,6 +81,7 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::SentryHelper helpers ::API::Helpers + helpers ::API::Helpers::CommonHelpers # Keep in alphabetical order mount ::API::AccessRequests diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 409cb5b924f..9fcf04efa38 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -121,7 +121,7 @@ module API end def oauth2_bearer_token_error_handler - Proc.new do |e| + proc do |e| response = case e when MissingTokenError diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 66b37fd2bcc..621b9dcecd9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,7 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -140,7 +140,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b888ede6fe8..8a54f7f3f05 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -47,6 +47,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/deploy_keys" do params[:key].strip! diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5954aea8041..01cc8e8e1ca 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -5,7 +5,10 @@ module API end class UserBasic < UserSafe - expose :id, :state, :avatar_url + expose :id, :state + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -14,10 +17,15 @@ module API class User < UserBasic expose :created_at - expose :is_admin?, as: :is_admin expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +33,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity @@ -34,8 +43,9 @@ module API expose :external end - class UserWithPrivateToken < UserPublic + class UserWithPrivateDetails < UserPublic expose :private_token + expose :admin?, as: :is_admin end class Email < Grape::Entity @@ -43,14 +53,14 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events expose :enable_ssl_verification end class ProjectHook < Hook expose :project_id, :issues_events, :merge_requests_events expose :note_events, :pipeline_events, :wiki_page_events - expose :build_events, as: :job_events + expose :job_events end class BasicProjectDetails < Grape::Entity @@ -90,7 +100,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -134,7 +146,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -184,19 +198,15 @@ module API end expose :protected do |repo_branch, options| - options[:project].protected_branch?(repo_branch.name) + ProtectedBranch.protected?(options[:project], repo_branch.name) end expose :developers_can_push do |repo_branch, options| - project = options[:project] - access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } + options[:project].protected_branches.developers_can?(:push, repo_branch.name) end expose :developers_can_merge do |repo_branch, options| - project = options[:project] - access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten - access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } + options[:project].protected_branches.developers_can?(:merge, repo_branch.name) end end @@ -204,7 +214,7 @@ module API expose :id, :name, :type, :path expose :mode do |obj, options| - filemode = obj.mode.to_s(8) + filemode = obj.mode filemode = "0" + filemode if filemode.length < 6 filemode end @@ -253,7 +263,11 @@ module API class IssueBasic < ProjectEntity expose :label_names, as: :labels expose :milestone, using: Entities::Milestone - expose :assignee, :author, using: Entities::UserBasic + expose :assignees, :author, using: Entities::UserBasic + + expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + issue.assignees.first + end expose :user_notes_count expose :upvotes, :downvotes @@ -456,7 +470,7 @@ module API expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events expose :tag_push_events, :note_events, :pipeline_events - expose :build_events, as: :job_events + expose :job_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -581,6 +595,7 @@ module API expose :plantuml_enabled expose :plantuml_url expose :terminal_max_session_time + expose :polling_interval_multiplier end class Release < Grape::Entity @@ -614,9 +629,9 @@ module API expose :locked expose :version, :revision, :platform, :architecture expose :contacted_at - expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } + expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } expose :projects, with: Entities::BasicProjectDetails do |runner, options| - if options[:current_user].is_admin? + if options[:current_user].admin? runner.projects else options[:current_user].authorized_projects.where(id: runner.projects) diff --git a/lib/api/files.rb b/lib/api/files.rb index 33fc970dc09..e6ea12c5ab7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,7 +5,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -130,7 +130,7 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] != :success render_api_error!(result[:message], 400) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 8f3799417e3..3da7d735da8 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -5,11 +5,16 @@ module API before { authenticate! } helpers do - params :optional_params do + params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' + end + + params :optional_params do + use :optional_params_ce end params :statistics_params do @@ -19,7 +24,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -47,7 +52,7 @@ module API elsif current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end @@ -56,7 +61,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.is_admin? + present_groups groups, statistics: params[:statistics] && current_user.admin? end desc 'Create a group. Available only for users who can create groups.' do @@ -142,7 +147,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 61527c1e20b..226a7ddd50e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,8 @@ module API end def find_project_snippet(id) - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params).find(id) + finder_params = { project: user_project } + SnippetsFinder.new(current_user, finder_params).execute.find(id) end def find_merge_request_with_access(iid, access_level = :read_merge_request) @@ -102,7 +102,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(current_user, :access_api) + unauthorized! unless current_user && can?(initial_current_user, :access_api) end def authenticate_non_get! @@ -118,7 +118,7 @@ module API def authenticated_as_admin! authenticate! - forbidden! unless current_user.is_admin? + forbidden! unless current_user.admin? end def authorize!(action, subject = :global) @@ -301,7 +301,7 @@ module API UploadedFile.new( file_path, params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', + params["#{field}.type"] || 'application/octet-stream' ) end @@ -358,7 +358,7 @@ module API return unless sudo_identifier return unless initial_current_user - unless initial_current_user.is_admin? + unless initial_current_user.admin? forbidden!('Must be admin to use sudo') end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb new file mode 100644 index 00000000000..322624c6092 --- /dev/null +++ b/lib/api/helpers/common_helpers.rb @@ -0,0 +1,13 @@ +module API + module Helpers + module CommonHelpers + def convert_parameters_from_legacy_format(params) + params.tap do |params| + if params[:assignee_id].present? + params[:assignee_ids] = [params.delete(:assignee_id)] + end + end + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 2135a787b11..264df7271a3 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,48 +1,14 @@ module API module Helpers module InternalHelpers - # Project paths may be any of the following: - # * /repository/storage/path/namespace/project - # * /namespace/project - # * namespace/project - # - # In addition, they may have a '.git' extension and multiple namespaces - # - # Transform all these cases to 'namespace/project' - def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values) - project_path = project_path.sub(/\.git\z/, '') - - storages.each do |storage| - storage_path = File.expand_path(storage['path']) - - if project_path.start_with?(storage_path) - project_path = project_path.sub(storage_path, '') - break - end - end - - project_path.sub(/\A\//, '') - end - - def project_path - @project_path ||= clean_project_path(params[:project]) - end - def wiki? - @wiki ||= project_path.end_with?('.wiki') && - !Project.find_by_full_path(project_path) + set_project unless defined?(@wiki) + @wiki end def project - @project ||= begin - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - Project.find_by_full_path(project_path) - end + set_project unless defined?(@project) + @project end def ssh_authentication_abilities @@ -53,12 +19,28 @@ module API ] end - def parse_allowed_environment_variables - return if params[:env].blank? + def parse_env + return {} if params[:env].blank? JSON.parse(params[:env]) - rescue JSON::ParserError + {} + end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end + + private + + def set_project + if params[:gl_repository] + @project, @wiki = Gitlab::GlRepository.parse(params[:gl_repository]) + else + @project, @wiki = Gitlab::RepoPath.parse(params[:project]) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 74848a6e144..1369b021ea4 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -50,10 +50,14 @@ module API forbidden!('Job has been erased!') if job.erased? end - def authenticate_job!(job) + def authenticate_job! + job = Ci::Build.find_by_id(params[:id]) + validate_job!(job) do forbidden! unless job_token_valid?(job) end + + job end def job_token_valid?(job) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 523f38d129e..96aaaf868ea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -11,14 +11,16 @@ module API # Params: # key_id - ssh key id for Git over SSH # user_id - user id for Git over HTTP + # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project path with namespace # action - git action (git-upload-pack or git-receive-pack) - # ref - branch name - # forced_push - forced_push - # protocol - Git access protocol being used, e.g. HTTP or SSH + # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList post "/allowed" do status 200 + # Stores some Git-specific env thread-safely + Gitlab::Git::Env.set(parse_env) + actor = if params[:key_id] Key.find_by(id: params[:key_id]) @@ -30,22 +32,20 @@ module API actor.update_last_used_at if actor.is_a?(Key) - access = - if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) - else - Gitlab::GitAccess.new(actor, - project, - protocol, - authentication_abilities: ssh_authentication_abilities, - env: parse_allowed_environment_variables) - end - - access_status = access.check(params[:action], params[:changes]) + access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_status = access_checker + .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + .check(params[:action], params[:changes]) response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = @@ -90,7 +90,7 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION, + gitlab_rev: Gitlab::REVISION } end @@ -139,9 +139,10 @@ module API return unless Gitlab::GitalyClient.enabled? begin - Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive + repository = wiki? ? project.wiki.repository : project.repository + Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive rescue GRPC::Unavailable => e - render_api_error(e, 500) + render_api_error!(e, 500) end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4dce5dd130a..78db960ae28 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -26,17 +26,23 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' + optional :search, type: String, desc: 'Search issues for text present in the title or description' use :pagination end - params :issue_params do + params :issue_params_ce do optional :description, type: String, desc: 'The description of an issue' - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' + optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end + + params :issue_params do + use :issue_params_ce + end end resource :issues do @@ -130,6 +136,8 @@ module API issue_params = declared_params(include_missing: false) + issue_params = convert_parameters_from_legacy_format(issue_params) + issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute @@ -154,7 +162,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do @@ -168,6 +176,8 @@ module API update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) + issue = ::Issues::UpdateService.new(user_project, current_user, update_params).execute(issue) @@ -214,6 +224,21 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + desc 'List merge requests closing issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/closed_by' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id) + merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids) + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end end end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ffab0aafe59..0223957fde1 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -118,7 +118,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end @@ -132,6 +132,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) build.cancel @@ -148,6 +149,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -165,6 +167,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -181,6 +184,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -201,6 +205,7 @@ module API build = get_build!(params[:job_id]) + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -211,12 +216,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c8033664133..710deba5ae3 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -20,6 +20,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -33,13 +35,28 @@ module API end end - params :optional_params do + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' end + + params :optional_params do + use :optional_params_ce + end end desc 'List merge requests' do @@ -53,23 +70,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end - - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end @@ -145,14 +154,24 @@ module API success Entities::MergeRequest end params do + # CE + at_least_one_of_ce = [ + :assignee_id, + :description, + :labels, + :milestone_id, + :remove_source_branch, + :state_event, + :target_branch, + :title + ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], desc: 'Status of the merge request' + use :optional_params - at_least_one_of :title, :target_branch, :description, :assignee_id, - :milestone_id, :labels, :state_event, - :remove_source_branch + at_least_one_of(*at_least_one_of_ce) end put ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) @@ -173,6 +192,7 @@ module API success Entities::MergeRequest end params do + # CE optional :merge_commit_message, type: String, desc: 'Custom merge commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' @@ -182,14 +202,15 @@ module API end put ':id/merge_requests/:merge_request_iid/merge' do merge_request = find_project_merge_request(params[:merge_request_iid]) + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! unless merge_request.mergeable_state? + not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) if params[:sha] && merge_request.diff_head_sha != params[:sha] render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) @@ -200,7 +221,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index e7ab82f08db..a3ea619a2fb 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -139,7 +139,7 @@ module API finder_params = { project_id: user_project.id, - milestone_id: milestone.id, + milestone_title: milestone.title, sort: 'position_asc' } diff --git a/lib/api/notes.rb b/lib/api/notes.rb index de39e579ac3..e281e3230fd 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -78,7 +78,7 @@ module API } if can?(current_user, noteable_read_ability_name(noteable), noteable) - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 754c3d85a04..9117704aa46 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -14,13 +14,23 @@ module API end params do use :pagination - optional :scope, type: String, values: %w(running branches tags), - desc: 'Either running, branches, or tags' + optional :scope, type: String, values: %w[running pending finished branches tags], + desc: 'The scope of pipelines' + optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, + desc: 'The status of pipelines' + optional :ref, type: String, desc: 'The ref of pipelines' + optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' + optional :name, type: String, desc: 'The name of the user who triggered pipelines' + optional :username, type: String, desc: 'The username of the user who triggered pipelines' + optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', + desc: 'Order pipelines' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Sort pipelines' end get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 53791166c33..7a345289617 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -13,7 +13,7 @@ module API optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" - optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" @@ -53,7 +53,9 @@ module API use :project_hook_properties end post ":id/hooks" do - hook = user_project.hooks.new(declared_params(include_missing: false)) + hook_params = declared_params(include_missing: false) + + hook = user_project.hooks.new(hook_params) if hook.save present hook, with: Entities::ProjectHook @@ -74,7 +76,9 @@ module API put ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - if hook.update_attributes(declared_params(include_missing: false)) + update_params = declared_params(include_missing: false) + + if hook.update_attributes(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index cfee38a9baf..98bc9c28527 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -17,8 +17,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0fbe1669d45..ed5004e8d1a 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -6,12 +6,12 @@ module API before { authenticate_non_get! } helpers do - params :optional_params do + params :optional_params_ce do optional :description, type: String, desc: 'The description of the project' optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -22,6 +22,14 @@ module API optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' end + + params :optional_params do + use :optional_params_ce + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end end resource :projects do @@ -52,10 +60,6 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' @@ -65,7 +69,7 @@ module API options = options.reverse_merge( with: Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -81,10 +85,11 @@ module API end params do use :collection_params + use :statistics_params end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] end desc 'Create new project' do @@ -99,6 +104,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -146,10 +152,13 @@ module API desc 'Get a single project' do success Entities::ProjectWithAccess end + params do + use :statistics_params + end get ":id" do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end desc 'Get events for a single project' do @@ -198,17 +207,33 @@ module API success Entities::Project end params do + # CE + at_least_one_of_ce = + [ + :jobs_enabled, + :container_registry_enabled, + :default_branch, + :description, + :issues_enabled, + :lfs_enabled, + :merge_requests_enabled, + :name, + :only_allow_merge_if_all_discussions_are_resolved, + :only_allow_merge_if_pipeline_succeeds, + :path, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :visibility, + :wiki_enabled + ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' optional :path, type: String, desc: 'The path of the repository' + use :optional_params - at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, - :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :visibility, :public_builds, - :request_access_enabled, :only_allow_merge_if_pipeline_succeeds, - :only_allow_merge_if_all_discussions_are_resolved, :path, - :default_branch + at_least_one_of(*at_least_one_of_ce) end put ':id' do authorize_admin_project @@ -216,6 +241,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success @@ -358,7 +385,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 4c9db2c8716..6fbb02cb3aa 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -113,10 +113,9 @@ module API optional :state, type: String, desc: %q(Job's status: success, failed) end put '/:id' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! - job.update_attributes(trace: params[:trace]) if params[:trace] + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.path_with_namespace) @@ -140,23 +139,20 @@ module API optional :token, type: String, desc: %q(Job's authentication token) end patch '/:id/trace' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = job.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = job.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - job.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Job-Status', job.status - header 'Range', "0-#{job.trace_length}" + header 'Range', "0-#{stream_size}" end desc 'Authorize artifacts uploading for job' do @@ -175,8 +171,7 @@ module API require_gitlab_workhorse! Gitlab::Workhorse.verify_api_request!(headers) - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! forbidden!('Job is not running') unless job.running? if params[:filesize] @@ -212,8 +207,7 @@ module API not_allowed! unless Gitlab.config.artifacts.enabled require_gitlab_workhorse! - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! forbidden!('Job is not running!') unless job.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path @@ -245,8 +239,7 @@ module API optional :token, type: String, desc: %q(Job's authentication token) end get '/:id/artifacts' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! artifacts_file = job.artifacts_file unless artifacts_file.file_storage? diff --git a/lib/api/runners.rb b/lib/api/runners.rb index a77c876a749..db6c7c59092 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -161,18 +161,18 @@ module API end def authenticate_show_runner!(runner) - return if runner.is_shared || current_user.is_admin? + return if runner.is_shared || current_user.admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_update_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_delete_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) @@ -181,7 +181,7 @@ module API def authenticate_enable_runner!(runner) forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner is locked") if runner.locked? - return if current_user.is_admin? + return if current_user.admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 4e0c9cb1f63..cb07df9e249 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -356,7 +356,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -488,6 +488,14 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + } + ], 'mattermost' => [ { required: true, @@ -550,7 +558,8 @@ module API RedmineService, SlackService, MattermostService, - TeamcityService, + MicrosoftTeamsService, + TeamcityService ] if Rails.env.development? @@ -562,8 +571,14 @@ module API desc: 'URL to the mock service' } ] + services['mock-deployment'] = [] + services['mock-monitoring'] = [] - service_classes << MockCiService + service_classes += [ + MockCiService, + MockDeploymentService, + MockMonitoringService + ] end trigger_services = { @@ -627,7 +642,7 @@ module API service_params = declared_params(include_missing: false).merge(active: true) if service.update_attributes(service_params) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? else render_api_error!('400 Bad Request', 400) end @@ -658,7 +673,7 @@ module API end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? end end diff --git a/lib/api/session.rb b/lib/api/session.rb index 002ffd1d154..016415c3023 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserWithPrivateToken + success Entities::UserWithPrivateDetails end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateToken + present user, with: Entities::UserWithPrivateDetails end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d4d3229f0d1..82f513c984e 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -20,6 +20,56 @@ module API success Entities::ApplicationSetting end params do + # CE + at_least_one_of_ce = [ + :admin_notification_email, + :after_sign_out_path, + :after_sign_up_text, + :akismet_enabled, + :container_registry_token_expire_delay, + :default_artifacts_expire_in, + :default_branch_protection, + :default_group_visibility, + :default_project_visibility, + :default_projects_limit, + :default_snippet_visibility, + :disabled_oauth_sign_in_sources, + :domain_blacklist_enabled, + :domain_whitelist, + :email_author_in_body, + :enabled_git_access_protocol, + :gravatar_enabled, + :help_page_text, + :home_page_url, + :housekeeping_enabled, + :html_emails_enabled, + :import_sources, + :koding_enabled, + :max_artifacts_size, + :max_attachment_size, + :max_pages_size, + :metrics_enabled, + :plantuml_enabled, + :polling_interval_multiplier, + :recaptcha_enabled, + :repository_checks_enabled, + :repository_storage, + :require_two_factor_authentication, + :restricted_visibility_levels, + :send_user_confirmation_email, + :sentry_enabled, + :clientside_sentry_enabled, + :session_expire_delay, + :shared_runners_enabled, + :sidekiq_throttling_enabled, + :sign_in_text, + :signin_enabled, + :signup_enabled, + :terminal_max_session_time, + :user_default_external, + :user_oauth_applications, + :version_check_enabled + ] optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' @@ -89,6 +139,10 @@ module API given sentry_enabled: ->(val) { val } do requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' end + optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' + given clientside_sentry_enabled: ->(val) { val } do + requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' + end optional :repository_storage, type: String, desc: 'Storage paths for new projects' optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :koding_enabled, type: Boolean, desc: 'Enable Koding' @@ -110,22 +164,9 @@ module API requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' - at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, - :default_group_visibility, :restricted_visibility_levels, :import_sources, - :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, - :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, - :user_oauth_applications, :user_default_external, :signup_enabled, - :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, - :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, - :shared_runners_enabled, :max_artifacts_size, - :default_artifacts_expire_in, :max_pages_size, - :container_registry_token_expire_delay, - :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, - :akismet_enabled, :admin_notification_email, :sentry_enabled, - :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, - :version_check_enabled, :email_author_in_body, :html_emails_enabled, - :housekeeping_enabled, :terminal_max_session_time + optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + + at_least_one_of(*at_least_one_of_ce) end put "application/settings" do attrs = declared_params(include_missing: false) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b93fdc62808..53f5953a8fb 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index dbe54d3cd31..91567909998 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -5,7 +5,7 @@ module API subscribable_types = { 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/users.rb b/lib/api/users.rb index a4201fe6fed..3d83720b7b9 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -37,11 +37,16 @@ module API success Entities::UserBasic end params do + # CE optional :username, type: String, desc: 'Get a single user with a specific username' + optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + all_or_none_of :extern_uid, :provider + use :pagination end get do @@ -49,17 +54,11 @@ module API render_api_error!("Not authorized.", 403) end - if params[:username].present? - users = User.where(username: params[:username]) - else - users = User.all - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - users = users.external if params[:external] && current_user.is_admin? - end + authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) - entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic + users = UsersFinder.new(current_user, params).execute + + entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic present paginate(users), with: entity end @@ -73,7 +72,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - if current_user && current_user.is_admin? + if current_user && current_user.admin? present user, with: Entities::UserPublic elsif can?(current_user, :read_user, user) present user, with: Entities::User @@ -293,7 +292,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - ::Users::DestroyService.new(current_user).execute(user) + DeleteUserWorker.perform_async(current_user.id, user.id) end desc 'Block a user. Available only for admins.' @@ -341,7 +340,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent @@ -425,7 +424,7 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do @@ -532,6 +531,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 6f97102c6ef..21935922414 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -120,7 +120,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end @@ -134,6 +134,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) build.cancel @@ -150,6 +151,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -167,6 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -183,6 +186,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -202,7 +206,7 @@ module API authorize_read_builds! build = get_build!(params[:build_id]) - + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -213,12 +217,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 3414a2883e5..674de592f0a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -53,7 +53,7 @@ module API attrs = declared_params.dup branch = attrs.delete(:branch_name) - attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + attrs.merge!(start_branch: branch, branch_name: branch) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -131,7 +131,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 832b4bdeb4f..332f233bf5e 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -69,7 +69,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -129,7 +131,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -234,7 +238,8 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :build_events, :pipeline_events + expose :tag_push_events, :note_events, :pipeline_events + expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -246,7 +251,15 @@ module API class ProjectHook < ::API::Entities::Hook expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :build_events, :pipeline_events, :wiki_page_events + expose :note_events, :pipeline_events, :wiki_page_events + expose :job_events, as: :build_events + end + + class Issue < ::API::Entities::Issue + unexpose :assignees + expose :assignee do |issue, options| + ::API::Entities::UserBasic.represent(issue.assignees.first, options) + end end end end diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb index 13542b0c71c..c76acc86504 100644 --- a/lib/api/v3/files.rb +++ b/lib/api/v3/files.rb @@ -6,7 +6,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -123,7 +123,7 @@ module API file_params = declared_params(include_missing: false) file_params[:branch] = file_params.delete(:branch_name) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index c5b37622d79..6187445fc8d 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -20,7 +20,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -45,7 +45,7 @@ module API groups = if current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end @@ -54,7 +54,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.is_admin? + present_groups groups, statistics: params[:statistics] && current_user.admin? end desc 'Get list of owned groups for authenticated user' do @@ -151,7 +151,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index 715083fc4f8..cb371fdbab8 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -8,6 +8,7 @@ module API helpers do def find_issues(args = {}) args = params.merge(args) + args = convert_parameters_from_legacy_format(args) args.delete(:id) args[:milestone_title] = args.delete(:milestone) @@ -51,7 +52,7 @@ module API resource :issues do desc "Get currently authenticated user's issues" do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -61,7 +62,7 @@ module API get do issues = find_issues(scope: 'authored') - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -70,7 +71,7 @@ module API end resource :groups, requirements: { id: %r{[^/]+} } do desc 'Get a list of group issues' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -82,7 +83,7 @@ module API issues = find_issues(group_id: group.id, match_all_labels: true) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -94,7 +95,7 @@ module API desc 'Get a list of project issues' do detail 'iid filter is deprecated have been removed on V4' - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -107,22 +108,22 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Get a single project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' end get ":id/issues/:issue_id" do issue = find_project_issue(params[:issue_id]) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Create a new project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :title, type: String, desc: 'The title of an issue' @@ -140,6 +141,7 @@ module API issue_params = declared_params(include_missing: false) issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions)) + issue_params = convert_parameters_from_legacy_format(issue_params) issue = ::Issues::CreateService.new(user_project, current_user, @@ -147,14 +149,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Update an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -176,6 +178,7 @@ module API end update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) issue = ::Issues::UpdateService.new(user_project, current_user, @@ -184,14 +187,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Move an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -206,7 +209,7 @@ module API begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 3077240e650..b6b7254ae29 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -23,6 +23,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -32,7 +34,7 @@ module API if project.has_external_issue_tracker? ::API::Entities::ExternalIssue else - ::API::Entities::Issue + ::API::V3::Entities::Issue end end diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index be90cec4afc..4c7061d4939 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -39,7 +39,7 @@ module API end desc 'Get all issues for a single project milestone' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -56,7 +56,7 @@ module API } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end end end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb index 4f8e0eff4ff..009ec5c6bbd 100644 --- a/lib/api/v3/notes.rb +++ b/lib/api/v3/notes.rb @@ -79,7 +79,7 @@ module API noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) if can?(current_user, noteable_read_ability_name(noteable), noteable) - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index 82827249244..c48cbd2b765 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -21,7 +21,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute present paginate(pipelines), with: ::API::Entities::Pipeline end end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index fc065a22d74..c41fee32610 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -18,8 +18,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index b753dbab381..164612cb8dd 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -88,7 +88,7 @@ module API options = options.reverse_merge( with: ::API::V3::Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -107,7 +107,7 @@ module API end get '/visible' do entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity end desc 'Get a projects list for authenticated user' do @@ -452,7 +452,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index 1934d6e578c..faa265f3314 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -50,7 +50,7 @@ module API helpers do def authenticate_delete_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 3bacaeee032..118c6df6549 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -377,7 +377,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -501,6 +501,12 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + ], 'mattermost' => [ { required: true, @@ -596,7 +602,7 @@ module API end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? end end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 07dac7e9904..0762fc02d70 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb index 068750ec077..690768db82f 100644 --- a/lib/api/v3/subscriptions.rb +++ b/lib/api/v3/subscriptions.rb @@ -7,7 +7,7 @@ module API 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 5e18cecc431..f4cda3b2eba 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -138,7 +138,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4016ac76348..d97e5d98229 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -80,16 +80,32 @@ module Backup 'port' => '--port', 'socket' => '--socket', 'username' => '--user', - 'encoding' => '--default-character-set' + 'encoding' => '--default-character-set', + # SSL + 'sslkey' => '--ssl-key', + 'sslcert' => '--ssl-cert', + 'sslca' => '--ssl-ca', + 'sslcapath' => '--ssl-capath', + 'sslcipher' => '--ssl-cipher' } args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + args = { + 'username' => 'PGUSER', + 'host' => 'PGHOST', + 'port' => 'PGPORT', + 'password' => 'PGPASSWORD', + # SSL + 'sslmode' => 'PGSSLMODE', + 'sslkey' => 'PGSSLKEY', + 'sslcert' => 'PGSSLCERT', + 'sslrootcert' => 'PGSSLROOTCERT', + 'sslcrl' => 'PGSSLCRL', + 'sslcompression' => 'PGSSLCOMPRESSION' + } + args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] } end def report_success(success) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..330cd963626 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -15,11 +15,10 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(Gitlab.config.backup.path) do - File.open("#{Gitlab.config.backup.path}/backup_information.yml", - "w+") do |file| + Dir.chdir(backup_path) do + File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/, '') end @@ -64,9 +63,9 @@ module Backup $progress.print "Deleting tmp directories ... " backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + next unless File.exist?(File.join(backup_path, dir)) - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(backup_path, dir)) $progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) @@ -83,8 +82,8 @@ module Backup if keep_time > 0 removed = 0 - Dir.chdir(Gitlab.config.backup.path) do - Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + Dir.chdir(backup_path) do + backup_file_list.each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -107,18 +106,14 @@ module Backup end def unpack - Dir.chdir(Gitlab.config.backup.path) + Dir.chdir(backup_path) # check for existing backups in the backup dir - file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") - - if file_list.count == 0 - $progress.puts "No backups found in #{Gitlab.config.backup.path}" + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - end - - if file_list.count > 1 && ENV["BACKUP"].nil? + elsif backup_file_list.many? && ENV["BACKUP"].nil? $progress.puts 'Found more than one backup, please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 @@ -127,7 +122,7 @@ module Backup tar_file = if ENV['BACKUP'].present? "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" else - file_list.first + backup_file_list.first end unless File.exist?(tar_file) @@ -169,6 +164,14 @@ module Backup private + def backup_path + Gitlab.config.backup.path + end + + def backup_file_list + @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") + end + def connect_to_remote_directory(connection_settings) connection = ::Fog::Storage.new(connection_settings) diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d6138816e70..6255a611dbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -53,7 +53,10 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): + (?=[^[:alnum:]:]|$)/x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index d67d466bce8..d6327ef31cb 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -2,16 +2,17 @@ module Banzai module Filter # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter + SCHEMES = ['http', 'https', nil].freeze + def call links.each do |node| - href = href_to_lowercase_scheme(node["href"].to_s) + uri = uri(node['href'].to_s) + next unless uri - unless node["href"].to_s == href - node.set_attribute('href', href) - end + node.set_attribute('href', uri.to_s) - if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) - node.set_attribute('rel', 'nofollow noreferrer') + if SCHEMES.include?(uri.scheme) && external_url?(uri) + node.set_attribute('rel', 'nofollow noreferrer noopener') node.set_attribute('target', '_blank') end end @@ -21,27 +22,26 @@ module Banzai private + def uri(href) + URI.parse(href) + rescue URI::Error + nil + end + def links query = 'descendant-or-self::a[@href and not(@href = "")]' doc.xpath(query) end - def href_to_lowercase_scheme(href) - scheme_match = href.match(/\A(\w+):\/\//) - - if scheme_match - scheme_match.to_s.downcase + scheme_match.post_match - else - href - end - end + def external_url?(uri) + # Relative URLs miss a hostname + return false unless uri.hostname - def external_url?(url) - !url.start_with?(internal_url) + uri.hostname != internal_url.hostname end def internal_url - @internal_url ||= Gitlab.config.gitlab.url + @internal_url ||= URI.parse(Gitlab.config.gitlab.url) end end end diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb new file mode 100644 index 00000000000..327ea9449a1 --- /dev/null +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -0,0 +1,37 @@ +module Banzai + module Filter + # HTML filter that appends state information to issuable links. + # Runs as a post-process filter as issuable state might change whilst + # Markdown is in the cache. + # + # This filter supports cross-project references. + class IssuableStateFilter < HTML::Pipeline::Filter + VISIBLE_STATES = %w(closed merged).freeze + + def call + return doc unless context[:issuable_state_filter_enabled] + + extractor = Banzai::IssuableExtractor.new(project, current_user) + issuables = extractor.extract([doc]) + + issuables.each do |node, issuable| + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" + end + end + + doc + end + + private + + def current_user + context[:current_user] + end + + def project + context[:project] + end + end + end +end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ff580ec68f8..ee73fa91589 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -14,7 +14,7 @@ module Banzai def self.renderer @renderer ||= begin - renderer = Redcarpet::Render::HTML.new + renderer = Banzai::Renderer::HTML.new Redcarpet::Markdown.new(renderer, redcarpet_options) end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c59a80dd1c7..9f9882b3b40 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,7 +7,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Redactor.new(project, current_user).redact([doc]) + Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction] doc end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d5f9e252f62..522217deae4 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -24,10 +24,6 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class v-pre) - whitelist[:attributes]['span'] = %w(class) - # Allow table alignment whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -52,9 +48,6 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) - # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(self.class.clean_spans) - whitelist end @@ -84,21 +77,6 @@ module Banzai end end end - - def clean_spans - lambda do |env| - node = env[:node] - - return unless node.name == 'span' - return unless node.has_attribute?('class') - - unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } - node.remove_attribute('class') - end - - { node_whitelist: [node] } - end - end end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9f09ca90697..7da565043d1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai end def highlight_node(node) - language = node.attr('class') + language = node.attr('lang') code = node.text css_classes = "code highlight" lexer = lexer_for(language) diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb new file mode 100644 index 00000000000..cbabf9156de --- /dev/null +++ b/lib/banzai/issuable_extractor.rb @@ -0,0 +1,40 @@ +module Banzai + # Extract references to issuables from multiple documents + + # This populates RequestStore cache used in Banzai::ReferenceParser::IssueParser + # and Banzai::ReferenceParser::MergeRequestParser + # Populating the cache should happen before processing documents one-by-one + # so we can avoid N+1 queries problem + + class IssuableExtractor + QUERY = %q( + descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] + [@data-reference-type="issue" or @data-reference-type="merge_request"] + ).freeze + + attr_reader :project, :user + + def initialize(project, user) + @project = project + @user = user + end + + # Returns Hash in the form { node => issuable_instance } + def extract(documents) + nodes = documents.flat_map do |document| + document.xpath(QUERY) + end + + issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) + merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) + + issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( + merge_request_parser.merge_requests_for_nodes(nodes) + ) + + # The project for the issue/MR might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes.select { |node, issuable| issuable.project } + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9f8eb0931b8..002a3341ccd 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -31,7 +31,8 @@ module Banzai # # Returns the same input objects. def render(objects, attribute) - documents = render_objects(objects, attribute) + documents = render_documents(objects, attribute) + documents = post_process_documents(documents, objects, attribute) redacted = redact_documents(documents) objects.each_with_index do |object, index| @@ -41,9 +42,24 @@ module Banzai end end - # Renders the attribute of every given object. - def render_objects(objects, attribute) - render_attributes(objects, attribute) + private + + def render_documents(objects, attribute) + pipeline = HTML::Pipeline.new([]) + + objects.map do |object| + pipeline.to_document(Banzai.render_field(object, attribute)) + end + end + + def post_process_documents(documents, objects, attribute) + # Called here to populate cache, refer to IssuableExtractor docs + IssuableExtractor.new(project, user).extract(documents) + + documents.zip(objects).map do |document, object| + context = context_for(object, attribute) + Banzai::Pipeline[:post_process].to_document(document, context) + end end # Redacts the list of documents. @@ -57,25 +73,15 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.dup - context = context.merge(object.banzai_render_context(attribute)) - context - end - - # Renders the attributes of a set of objects. - # - # Returns an Array of `Nokogiri::HTML::Document`. - def render_attributes(objects, attribute) - objects.map do |object| - string = Banzai.render_field(object, attribute) - context = context_for(object, attribute) - - Banzai::Pipeline[:relative_link].to_document(string, context) - end + base_context.merge(object.banzai_render_context(attribute)) end def base_context - @base_context ||= @redaction_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge( + current_user: user, + project: project, + skip_redaction: true + ) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index fd4a6a107c2..bd4d1aa9ff8 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -9,9 +9,9 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ - Filter::SyntaxHighlightFilter, Filter::PlantumlFilter, Filter::SanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::UploadLinkFilter, diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb new file mode 100644 index 00000000000..c56d908009f --- /dev/null +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -0,0 +1,13 @@ +module Banzai + module Pipeline + class MarkupPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::PlantumlFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index ecff094b1e5..131ac3b0eec 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -4,6 +4,7 @@ module Banzai def self.filters FilterArray[ Filter::RelativeLinkFilter, + Filter::IssuableStateFilter, Filter::RedactorFilter ] end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 52fdb9a2140..c2503fa2adc 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -62,8 +62,7 @@ module Banzai nodes.select do |node| if node.has_attribute?(project_attr) - node_id = node.attr(project_attr).to_i - can_read_reference?(user, projects[node_id]) + can_read_reference?(user, projects[node]) else true end @@ -112,12 +111,12 @@ module Banzai per_project end - # Returns a Hash containing objects for an attribute grouped per their - # IDs. + # Returns a Hash containing objects for an attribute grouped per the + # nodes that reference them. # # The returned Hash uses the following format: # - # { id value => row } + # { node => row } # # nodes - An Array of HTML nodes to process. # @@ -132,9 +131,15 @@ module Banzai return {} if nodes.empty? ids = unique_attribute_values(nodes, attribute) - rows = collection_objects_for_ids(collection, ids) + collection_objects = collection_objects_for_ids(collection, ids) + objects_by_id = collection_objects.index_by(&:id) - rows.index_by(&:id) + nodes.each_with_object({}) do |node, hash| + if node.has_attribute?(attribute) + obj = objects_by_id[node.attr(attribute).to_i] + hash[node] = obj if obj + end + end end # Returns an Array containing all unique values of an attribute of the @@ -201,7 +206,7 @@ module Banzai # # The returned Hash uses the following format: # - # { project ID => project } + # { node => project } # def projects_for_nodes(nodes) @projects_for_nodes ||= diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 6c20dec5734..89ec715ddf6 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -13,14 +13,14 @@ module Banzai issues_readable_by_user(issues.values, user).to_set nodes.select do |node| - readable_issues.include?(issue_for_node(issues, node)) + readable_issues.include?(issues[node]) end end def referenced_by(nodes) issues = issues_for_nodes(nodes) - nodes.map { |node| issue_for_node(issues, node) }.uniq + nodes.map { |node| issues[node] }.compact.uniq end def issues_for_nodes(nodes) @@ -28,7 +28,7 @@ module Banzai nodes, Issue.all.includes( :author, - :assignee, + :assignees, { # These associations are primarily used for checking permissions. # Eager loading these ensures we don't end up running dozens of @@ -44,12 +44,6 @@ module Banzai self.class.data_attribute ) end - - private - - def issue_for_node(issues, node) - issues[node.attr(self.class.data_attribute).to_i] - end end end end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 40451947e6c..8b0662749fd 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -3,14 +3,42 @@ module Banzai class MergeRequestParser < BaseParser self.reference_type = :merge_request - def references_relation - MergeRequest.includes(:author, :assignee, :target_project) + def nodes_visible_to_user(user, nodes) + merge_requests = merge_requests_for_nodes(nodes) + + nodes.select do |node| + merge_request = merge_requests[node] + + merge_request && can?(user, :read_merge_request, merge_request.project) + end end - private + def referenced_by(nodes) + merge_requests = merge_requests_for_nodes(nodes) + + nodes.map { |node| merge_requests[node] }.compact.uniq + end - def can_read_reference?(user, ref_project) - can?(user, :read_merge_request, ref_project) + def merge_requests_for_nodes(nodes) + @merge_requests_for_nodes ||= grouped_objects_for_nodes( + nodes, + MergeRequest.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + target_project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members, + :project_feature + ] + }), + self.class.data_attribute + ) end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 7adaffa19c1..09b66cbd8fb 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -49,7 +49,7 @@ module Banzai # Check if project belongs to a group which # user can read. def can_read_group_reference?(node, user, groups) - node_group = groups[node.attr('data-group').to_i] + node_group = groups[node] node_group && can?(user, :read_group, node_group) end @@ -74,8 +74,8 @@ module Banzai if project && project_id && project.id == project_id.to_i true elsif project_id && user_id - project = projects[project_id.to_i] - user = users[user_id.to_i] + project = projects[node] + user = users[node] project && user ? project.team.member?(user) : false else diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb new file mode 100644 index 00000000000..252caa35947 --- /dev/null +++ b/lib/banzai/renderer/html.rb @@ -0,0 +1,13 @@ +module Banzai + module Renderer + class HTML < Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + end + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index 94adaacc9b5..800d5a075c6 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -1,6 +1,8 @@ module Bitbucket module Representation class Base + attr_reader :raw + def initialize(raw) @raw = raw end @@ -8,10 +10,6 @@ module Bitbucket def self.decorate(entries) entries.map { |entry| new(entry)} end - - private - - attr_reader :raw end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b3ccad7b28d..55402101e43 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -20,7 +20,7 @@ module Ci italic: 0x02, underline: 0x04, conceal: 0x08, - cross: 0x10, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) @@ -132,34 +132,54 @@ module Ci STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - def convert(raw, new_state) + def convert(stream, new_state) reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset open_new_tag - s = StringScanner.new(ansi) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size end - @offset += s.matched_size end close_open_tags() - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) end def handle_sequence(s) @@ -240,10 +260,10 @@ module Ci Base64.urlsafe_encode64(state.to_json) end - def restore_state(raw, new_state) + def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length + return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| send("#{param}=".to_sym, state[param]) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 746e76a1b1f..67b269b330c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -61,7 +61,7 @@ module Ci update_runner_info - build.update_attributes(trace: params[:trace]) if params[:trace] + build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: build.project.path_with_namespace) @@ -86,23 +86,20 @@ module Ci # Example Request: # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = build.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = build.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - build.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Build-Status', build.status - header 'Range', "0-#{build.trace_length}" + header 'Range', "0-#{stream_size}" end # Authorize artifacts uploading for build - Runners only @@ -117,8 +114,7 @@ module Ci require_gitlab_workhorse! Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! forbidden!('build is not running') unless build.running? if params[:filesize] @@ -154,8 +150,7 @@ module Ci post ":id/artifacts" do require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! forbidden!('Build is not running!') unless build.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path @@ -189,8 +184,7 @@ module Ci # Example Request: # GET /builds/:id/artifacts get ":id/artifacts" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -214,8 +208,7 @@ module Ci # Example Request: # DELETE /builds/:id/artifacts delete ":id/artifacts" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! status(200) build.erase_artifacts! diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 996990b464f..5109dc9670f 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,10 +13,14 @@ module Ci forbidden! unless current_runner end - def authenticate_build!(build) + def authenticate_build! + build = Ci::Build.find_by_id(params[:id]) + validate_build!(build) do forbidden! unless build_token_valid?(build) end + + build end def validate_build!(build) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 15a461a16dd..b06474cda7f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -70,7 +70,7 @@ module Ci cache: job[:cache], dependencies: job[:dependencies], after_script: job[:after_script], - environment: job[:environment], + environment: job[:environment] }.compact } end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index bae4db1ca4d..5f379756c11 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,16 +2,8 @@ class GroupUrlConstrainer def matches?(request) id = request.params[:id] - return false unless valid?(id) + return false unless DynamicPathValidator.valid?(id) - Group.find_by_full_path(id).present? - end - - private - - def valid?(id) - id.split('/').all? do |namespace| - NamespaceValidator.valid?(namespace) - end + Group.find_by_full_path(id, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index a10b4657d7d..6f542f63f98 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,10 +4,8 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = namespace_path + '/' + project_path - unless ProjectPathValidator.valid?(project_path) - return false - end + return false unless DynamicPathValidator.valid?(full_path) - Project.find_by_full_path(full_path).present? + Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 9ab5bcb12ff..28159dc0dec 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,5 +1,5 @@ class UserUrlConstrainer def matches?(request) - User.find_by_username(request.params[:username]).present? + User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present? end end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..d5f85f9fcad 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name, digest) + client.delete_blob(repository.path, digest) end def data - @data ||= client.blob(repository.name, digest, type) + @data ||= client.blob(repository.path, digest, type) end end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 7f5f6d9ddb6..c7263f302ab 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -75,10 +75,7 @@ module ContainerRegistry def redirect_response(location) return unless location - # We explicitly remove authorization token - faraday_blob.get(location) do |req| - req['Authorization'] = '' - end + faraday_redirect.get(location) end def faraday @@ -93,5 +90,14 @@ module ContainerRegistry initialize_connection(conn, @options) end end + + # Create a new request to make sure the Authorization header is not inserted + # via the Faraday middleware + def faraday_redirect + @faraday_redirect ||= Faraday.new(@base_uri) do |conn| + conn.request :json + conn.adapter :net_http + end + end end end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb new file mode 100644 index 00000000000..61849a40383 --- /dev/null +++ b/lib/container_registry/path.rb @@ -0,0 +1,76 @@ +module ContainerRegistry + ## + # Class responsible for extracting project and repository name from + # image repository path provided by a containers registry API response. + # + # Example: + # + # some/group/my_project/my/image -> + # project: some/group/my_project + # repository: my/image + # + class Path + InvalidRegistryPathError = Class.new(StandardError) + + LEVELS_SUPPORTED = 3 + + def initialize(path) + @path = path.to_s.downcase + end + + def valid? + @path =~ Gitlab::Regex.container_repository_name_regex && + components.size > 1 && + components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def components + @components ||= @path.split('/') + end + + def nodes + raise InvalidRegistryPathError unless valid? + + @nodes ||= components.size.downto(2).map do |length| + components.take(length).join('/') + end + end + + def has_project? + repository_project.present? + end + + def has_repository? + return false unless has_project? + + repository_project.container_repositories + .where(name: repository_name).any? + end + + def root_repository? + @path == project_path + end + + def repository_project + @project ||= Project + .where_full_path_in(nodes.first(LEVELS_SUPPORTED)) + .first + end + + def repository_name + return unless has_project? + + @path.remove(%r(^#{Regexp.escape(project_path)}/?)) + end + + def project_path + return unless has_project? + + repository_project.full_path.downcase + end + + def to_s + @path + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 0e634f6b6ef..63bce655f57 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -8,10 +8,6 @@ module ContainerRegistry @client = ContainerRegistry::Client.new(uri, options) end - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - private def default_path diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..728deea224f 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,15 +22,17 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.path, name) end def path "#{repository.path}:#{name}" end + def location + "#{repository.location}:#{name}" + end + def [](key) return unless manifest @@ -38,9 +40,7 @@ module ContainerRegistry end def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.name, name) + @digest ||= client.repository_tag_digest(repository.path, name) end def config_blob @@ -82,7 +82,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.path, digest) end end end diff --git a/lib/github/client.rb b/lib/github/client.rb new file mode 100644 index 00000000000..e65d908d232 --- /dev/null +++ b/lib/github/client.rb @@ -0,0 +1,23 @@ +module Github + class Client + attr_reader :connection, :rate_limit + + def initialize(options) + @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, 60) + faraday.options.timeout = options.fetch(:timeout, 60) + faraday.authorization 'token', options.fetch(:token) + faraday.adapter :net_http + end + + @rate_limit = RateLimit.new(connection) + end + + def get(url, query = {}) + exceed, reset_in = rate_limit.get + sleep reset_in if exceed + + Github::Response.new(connection.get(url, query)) + end + end +end diff --git a/lib/github/collection.rb b/lib/github/collection.rb new file mode 100644 index 00000000000..014b2038c4b --- /dev/null +++ b/lib/github/collection.rb @@ -0,0 +1,29 @@ +module Github + class Collection + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch(url, query = {}) + return [] if url.blank? + + Enumerator.new do |yielder| + loop do + response = client.get(url, query) + response.body.each { |item| yielder << item } + + raise StopIteration unless response.rels.key?(:next) + url = response.rels[:next] + end + end.lazy + end + + private + + def client + @client ||= Github::Client.new(options) + end + end +end diff --git a/lib/github/error.rb b/lib/github/error.rb new file mode 100644 index 00000000000..66d7afaa787 --- /dev/null +++ b/lib/github/error.rb @@ -0,0 +1,3 @@ +module Github + RepositoryFetchError = Class.new(StandardError) +end diff --git a/lib/github/import.rb b/lib/github/import.rb new file mode 100644 index 00000000000..9c7eb965f93 --- /dev/null +++ b/lib/github/import.rb @@ -0,0 +1,412 @@ +require_relative 'error' + +module Github + class Import + include Gitlab::ShellAdapter + + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Note < ::Note + self.table_name = 'notes' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose + + def initialize(project, options) + @project = project + @repository = project.repository + @repo = project.import_source + @options = options + @verbose = options.fetch(:verbose, false) + @cached = Hash.new { |hash, key| hash[key] = Hash.new } + @errors = [] + end + + # rubocop: disable Rails/Output + def execute + puts 'Fetching repository...'.color(:aqua) if verbose + fetch_repository + puts 'Fetching labels...'.color(:aqua) if verbose + fetch_labels + puts 'Fetching milestones...'.color(:aqua) if verbose + fetch_milestones + puts 'Fetching pull requests...'.color(:aqua) if verbose + fetch_pull_requests + puts 'Fetching issues...'.color(:aqua) if verbose + fetch_issues + puts 'Cloning wiki repository...'.color(:aqua) if verbose + fetch_wiki_repository + puts 'Expiring repository cache...'.color(:aqua) if verbose + expire_repository_cache + + true + rescue Github::RepositoryFetchError + false + ensure + keep_track_of_errors + end + + private + + def fetch_repository + begin + project.create_repository unless project.repository.exists? + project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.set_remote_as_mirror('github') + project.repository.fetch_remote('github', forced: true) + rescue Gitlab::Shell::Error => e + error(:project, "https://github.com/#{repo}.git", e.message) + raise Github::RepositoryFetchError + end + end + + def fetch_wiki_repository + wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git" + wiki_path = "#{project.path_with_namespace}.wiki" + + unless project.wiki.repository_exists? + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors(:wiki, wiki_url, e.message) + end + end + + def fetch_labels + url = "/repos/#{repo}/labels" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + begin + representation = Github::Representation::Label.new(raw) + + label = project.labels.find_or_create_by!(title: representation.title) do |label| + label.color = representation.color + end + + cached[:label_ids][label.title] = label.id + rescue => e + error(:label, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_milestones + url = "/repos/#{repo}/milestones" + + while url + response = Github::Client.new(options).get(url, state: :all) + + response.body.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + next if project.milestones.where(iid: milestone.iid).exists? + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_pull_requests + url = "/repos/#{repo}/pulls" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) + next unless merge_request.new_record? && pull_request.valid? + + begin + restore_branches(pull_request) + + author_id = user_id(pull_request.author, project.creator_id) + description = format_description(pull_request.description, pull_request.author) + + merge_request.attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: pull_request.source_project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: pull_request.target_project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + milestone_id: milestone_id(pull_request.milestone), + author_id: author_id, + assignee_id: user_id(pull_request.assignee), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + merge_request.save!(validate: false) + merge_request.merge_request_diffs.create + + # Fetch review comments + review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" + fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) + + # Fetch comments + comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" + fetch_comments(merge_request, :comment, comments_url) + rescue => e + error(:pull_request, pull_request.url, e.message) + ensure + clean_up_restored_branches(pull_request) + end + end + + url = response.rels[:next] + end + end + + def fetch_issues + url = "/repos/#{repo}/issues" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + next unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) + end + end + rescue => e + error(:issue, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_comments(noteable, type, url, klass = Note) + while url + comments = Github::Client.new(options).get(url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + representation = Github::Representation::Comment.new(raw, options) + author_id = user_id(representation.author, project.creator_id) + + note = klass.new + note.project_id = project.id + note.noteable = noteable + note.note = format_description(representation.note, representation.author) + note.commit_id = representation.commit_id + note.line_code = representation.line_code + note.author_id = author_id + note.created_at = representation.created_at + note.updated_at = representation.updated_at + note.save!(validate: false) + rescue => e + error(type, representation.url, e.message) + end + end + end + + url = comments.rels[:next] + end + end + + def fetch_releases + url = "/repos/#{repo}/releases" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + representation = Github::Representation::Release.new(raw) + next unless representation.valid? + + release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) + next unless relese.new_record? + + begin + release.description = representation.description + release.created_at = representation.created_at + release.updated_at = representation.updated_at + release.save!(validate: false) + rescue => e + error(:release, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def restore_branches(pull_request) + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + end + + def restore_source_branch(pull_request) + repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def label_ids(labels) + labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + end + + def milestone_id(milestone) + return unless milestone.present? + + project.milestones.select(:id).find_by(iid: milestone.iid)&.id + end + + def user_id(user, fallback_id = nil) + return unless user.present? + return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) + + gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) + + cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? + cached[:user_ids][user.id] = gitlab_user_id || fallback_id + end + + def user_id_by_email(email) + return nil unless email + + ::User.find_by_any_email(email)&.id + end + + def user_id_by_external_uid(id) + return nil unless id + + ::User.select(:id) + .joins(:identities) + .merge(::Identity.where(provider: :github, extern_uid: id)) + .first&.id + end + + def format_description(body, author) + return body if cached[:gitlab_user_ids][author.id] + + "*Created by: #{author.username}*\n\n#{body}" + end + + def expire_repository_cache + repository.expire_content_cache + end + + def keep_track_of_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def error(type, url, message) + errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } + end + end +end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb new file mode 100644 index 00000000000..884693d093c --- /dev/null +++ b/lib/github/rate_limit.rb @@ -0,0 +1,27 @@ +module Github + class RateLimit + SAFE_REMAINING_REQUESTS = 100 + SAFE_RESET_TIME = 500 + RATE_LIMIT_URL = '/rate_limit'.freeze + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def get + response = connection.get(RATE_LIMIT_URL) + + # GitHub Rate Limit API returns 404 when the rate limit is disabled + return false unless response.status != 404 + + body = Oj.load(response.body, class_cache: false, mode: :compat) + remaining = body.dig('rate', 'remaining').to_i + reset_in = body.dig('rate', 'reset').to_i + exceed = remaining <= SAFE_REMAINING_REQUESTS + + [exceed, reset_in] + end + end +end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb new file mode 100644 index 00000000000..c1c9448f305 --- /dev/null +++ b/lib/github/repositories.rb @@ -0,0 +1,19 @@ +module Github + class Repositories + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch + Collection.new(options).fetch(repos_url) + end + + private + + def repos_url + '/user/repos' + end + end +end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb new file mode 100644 index 00000000000..f26bdbdd546 --- /dev/null +++ b/lib/github/representation/base.rb @@ -0,0 +1,30 @@ +module Github + module Representation + class Base + def initialize(raw, options = {}) + @raw = raw + @options = options + end + + def id + raw['id'] + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + private + + attr_reader :raw, :options + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb new file mode 100644 index 00000000000..d1dac6944f0 --- /dev/null +++ b/lib/github/representation/branch.rb @@ -0,0 +1,51 @@ +module Github + module Representation + class Branch < Representation::Base + attr_reader :repository + + def user + raw.dig('user', 'login') || 'unknown' + end + + def repo + return @repo if defined?(@repo) + + @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + end + + def ref + raw['ref'] + end + + def sha + raw['sha'] + end + + def short_sha + Commit.truncate_sha(sha) + end + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + private + + def branch_exists? + repository.branch_exists?(ref) + end + + def commit_exists? + repository.branch_names_contains(sha).include?(ref) + end + + def repository + @repository ||= options.fetch(:repository) + end + end + end +end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb new file mode 100644 index 00000000000..1b5be91461b --- /dev/null +++ b/lib/github/representation/comment.rb @@ -0,0 +1,42 @@ +module Github + module Representation + class Comment < Representation::Base + def note + raw['body'] || '' + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def commit_id + raw['commit_id'] + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + private + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw['diff_hunk'] + end + + def file_path + raw['path'] + end + end + end +end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb new file mode 100644 index 00000000000..9713b82615d --- /dev/null +++ b/lib/github/representation/issuable.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Issuable < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee'], options) + end + + def assigned? + raw['assignee'].present? + end + end + end +end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb new file mode 100644 index 00000000000..df3540a6e6c --- /dev/null +++ b/lib/github/representation/issue.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Issue < Representation::Issuable + def labels + raw['labels'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'opened' + end + + def has_comments? + raw['comments'] > 0 + end + + def has_labels? + labels.count > 0 + end + + def pull_request? + raw['pull_request'].present? + end + end + end +end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb new file mode 100644 index 00000000000..60aa51f9569 --- /dev/null +++ b/lib/github/representation/label.rb @@ -0,0 +1,13 @@ +module Github + module Representation + class Label < Representation::Base + def color + "##{raw['color']}" + end + + def title + raw['name'] + end + end + end +end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb new file mode 100644 index 00000000000..917e6394ad4 --- /dev/null +++ b/lib/github/representation/milestone.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Milestone < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['description'] + end + + def due_date + raw['due_on'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'active' + end + end + end +end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb new file mode 100644 index 00000000000..ac9c8283b4b --- /dev/null +++ b/lib/github/representation/pull_request.rb @@ -0,0 +1,78 @@ +module Github + module Representation + class PullRequest < Representation::Issuable + attr_reader :project + + delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + + def source_project + project + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def target_project + project + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def state + return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? + return 'closed' if raw['state'] == 'closed' + + 'opened' + end + + def opened? + state == 'opened' + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + private + + def project + @project ||= options.fetch(:project) + end + + def source_branch + @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" + end + + def target_branch + @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + end + end +end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb new file mode 100644 index 00000000000..e7e4b428c1a --- /dev/null +++ b/lib/github/representation/release.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Release < Representation::Base + def description + raw['body'] + end + + def tag + raw['tag_name'] + end + + def valid? + !raw['draft'] + end + end + end +end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb new file mode 100644 index 00000000000..6938aa7db05 --- /dev/null +++ b/lib/github/representation/repo.rb @@ -0,0 +1,6 @@ +module Github + module Representation + class Repo < Representation::Base + end + end +end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb new file mode 100644 index 00000000000..18591380e25 --- /dev/null +++ b/lib/github/representation/user.rb @@ -0,0 +1,15 @@ +module Github + module Representation + class User < Representation::Base + def email + return @email if defined?(@email) + + @email = Github::User.new(username, options).get.fetch('email', nil) + end + + def username + raw['login'] + end + end + end +end diff --git a/lib/github/response.rb b/lib/github/response.rb new file mode 100644 index 00000000000..761c524b553 --- /dev/null +++ b/lib/github/response.rb @@ -0,0 +1,25 @@ +module Github + class Response + attr_reader :raw, :headers, :status + + def initialize(response) + @raw = response + @headers = response.headers + @status = response.status + end + + def body + Oj.load(raw.body, class_cache: false, mode: :compat) + end + + def rels + links = headers['Link'].to_s.split(', ').map do |link| + href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures + + [name.to_sym, href] + end + + Hash[*links.flatten] + end + end +end diff --git a/lib/github/user.rb b/lib/github/user.rb new file mode 100644 index 00000000000..f88a29e590b --- /dev/null +++ b/lib/github/user.rb @@ -0,0 +1,24 @@ +module Github + class User + attr_reader :username, :options + + def initialize(username, options) + @username = username + @options = options + end + + def get + client.get(user_url).body + end + + private + + def client + @client ||= Github::Client.new(options) + end + + def user_url + "/users/#{username}" + end + end +end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 8c28009b9c6..4714ab18cc1 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -32,7 +32,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Master" => MASTER, + "Master" => MASTER } end @@ -47,7 +47,7 @@ module Gitlab guest: GUEST, reporter: REPORTER, developer: DEVELOPER, - master: MASTER, + master: MASTER } end @@ -60,7 +60,7 @@ module Gitlab "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, + "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL } end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d575367d81a..96d38f6daa0 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -14,30 +14,18 @@ module Gitlab # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref - # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter # - def self.render(input, context, asciidoc_opts = {}) - asciidoc_opts.reverse_merge!( - safe: :secure, - backend: :gitlab_html5, - attributes: [] - ) - asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + def self.render(input, context) + asciidoc_opts = { safe: :secure, + backend: :gitlab_html5, + attributes: DEFAULT_ADOC_ATTRS } + + context[:pipeline] = :markup plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) - - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..099c45dcfb7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,6 +37,9 @@ module Gitlab end def find_with_user_password(login, password) + # Avoid resource intensive login checks if password is not provided + return unless password.present? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -44,7 +47,7 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? + return unless Gitlab::LDAP::Config.enabled? Gitlab::LDAP::Authentication.login(login, password) else @@ -108,7 +111,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4efa20374a..5a6d9ae99a0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,7 +149,7 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description - merge_request = project.merge_requests.create( + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, @@ -168,7 +168,7 @@ module Gitlab import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue StandardError => e - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } end end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb new file mode 100644 index 00000000000..4fc9a075edc --- /dev/null +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -0,0 +1,138 @@ +# This class is not backed by a table in the main database. +# It loads the latest Pipeline for the HEAD of a repository, and caches that +# in Redis. +module Gitlab + module Cache + module Ci + class ProjectPipelineStatus + attr_accessor :sha, :status, :ref, :project, :loaded + + delegate :commit, to: :project + + def self.load_for_project(project) + new(project).tap do |status| + status.load_status + end + end + + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + + def self.update_for_pipeline(pipeline) + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed + end + + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) + @project = project + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache + end + + def has_status? + loaded? && sha.present? && status.present? + end + + def load_status + return if loaded? + + if has_cache? + load_from_cache + else + load_from_project + store_in_cache + end + + self.loaded = true + end + + def load_from_project + return unless commit + + self.sha = commit.sha + self.status = commit.status + self.ref = project.default_branch + end + + # We only cache the status for the HEAD commit of a project + # This status is rendered in project lists + def store_in_cache_if_needed + return delete_from_cache unless commit + return unless sha + return unless ref + + if commit.sha == sha && project.default_branch == ref + store_in_cache + end + end + + def load_from_cache + Gitlab::Redis.with do |redis| + self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + end + end + + def store_in_cache + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + end + end + + def delete_from_cache + Gitlab::Redis.with do |redis| + redis.del(cache_key) + end + end + + def has_cache? + return self.loaded unless self.loaded.nil? + + Gitlab::Redis.with do |redis| + redis.exists(cache_key) + end + end + + def loaded? + self.loaded + end + + def cache_key + self.class.cache_key_for_project(project) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index f34ed0f4cf2..3e0c30c33b7 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -5,7 +5,7 @@ module Gitlab Gitlab::ChatCommands::IssueShow, Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy, + Gitlab::ChatCommands::Deploy ].freeze def execute diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/chat_commands/presenters/issue_base.rb index 054f7f4be0c..25bc82994ba 100644 --- a/lib/gitlab/chat_commands/presenters/issue_base.rb +++ b/lib/gitlab/chat_commands/presenters/issue_base.rb @@ -22,7 +22,7 @@ module Gitlab [ { title: "Assignee", - value: @resource.assignee ? @resource.assignee.name : "_None_", + value: @resource.assignees.any? ? @resource.assignees.first.name : "_None_", short: true }, { diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index c85f79127bc..c984eb20606 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,24 +1,25 @@ module Gitlab module Checks class ChangeAccess - # protocol is currently used only in EE attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( - change, user_access:, project:, env: {}, skip_authorization: false, + change, user_access:, project:, skip_authorization: false, protocol: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) + @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @env = env @skip_authorization = skip_authorization @protocol = protocol end def exec - error = push_checks || tag_checks || protected_branch_checks + return GitAccessStatus.new(true) if skip_authorization + + error = push_checks || branch_checks || tag_checks if error GitAccessStatus.new(false, error) @@ -29,58 +30,95 @@ module Gitlab protected - def protected_branch_checks - return if skip_authorization + def push_checks + if user_access.cannot_do_action?(:push_code) + "You are not allowed to push code to this project." + end + end + + def branch_checks return unless @branch_name - return unless project.protected_branch?(@branch_name) + + if deletion? && @branch_name == project.default_branch + return "The default branch of a project cannot be deleted." + end + + protected_branch_checks + end + + def protected_branch_checks + return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) - return "You are not allowed to delete protected branches from this project." end + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + unless user_access.can_delete_branch?(@branch_name) + return 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.' + end + + unless protocol == 'web' + 'You can only delete protected branches using the web interface.' + end + end + + def protected_branch_push_checks if matching_merge_request? - if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) "You are not allowed to merge code into protected branches on this project." end else - if user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_push_to_branch?(@branch_name) "You are not allowed to push code to protected branches on this project." end end end def tag_checks - return if skip_authorization + return unless @tag_name - tag_ref = Gitlab::Git.tag_name(@ref) - - if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) - "You are not allowed to change existing tags on this project." + if tag_exists? && user_access.cannot_do_action?(:admin_project) + return "You are not allowed to change existing tags on this project." end + + protected_tag_checks end - def push_checks - return if skip_authorization + def protected_tag_checks + return unless ProtectedTag.protected?(project, @tag_name) - if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." + return "Protected tags cannot be updated." if update? + return "Protected tags cannot be deleted." if deletion? + + unless user_access.can_create_tag?(@tag_name) + return "You are not allowed to create this tag as it is protected." end end private - def protected_tag?(tag_name) - project.repository.tag_exists?(tag_name) + def tag_exists? + project.repository.tag_exists?(@tag_name) end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def update? + !Gitlab::Git.blank_ref?(@oldrev) && !deletion? + end + + def deletion? + Gitlab::Git.blank_ref?(@newrev) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index de0c9049ebf..1e73f89158d 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,20 +1,16 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev, env: {}) + def self.force_push?(project, oldrev, newrev) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute - - if exit_status == 0 - missed_ref.present? - else - raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." - end + Gitlab::Git::RevList.new( + path_to_repo: project.repository.path_to_repo, + oldrev: oldrev, newrev: newrev).missed_ref.present? end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 6f799c2f031..2e073334abc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -37,6 +37,12 @@ module Gitlab !directory? end + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + def has_parent? nodes > 0 end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 9b9a0a8125a..a78a85397bd 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -21,7 +21,13 @@ module Gitlab def validate_variables(variables) variables.is_a?(Hash) && - variables.all? { |key, value| validate_string(key) && validate_string(value) } + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) end def validate_string(value) diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c3b0e651c3a..8acab605c91 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -15,6 +15,10 @@ module Gitlab def self.default {} end + + def value + Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + end end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb new file mode 100644 index 00000000000..551483d0aaa --- /dev/null +++ b/lib/gitlab/ci/cron_parser.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + class CronParser + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze + VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze + + def initialize(cron, cron_timezone = 'UTC') + @cron = cron + @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name + end + + def next_time_from(time) + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? + end + + def cron_valid? + try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + end + + def cron_timezone_valid? + try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? + end + + private + + # NOTE: + # cron_timezone can only accept timezones listed in TZInfo::Timezone. + # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, + # because Rufus::Scheduler only supports TZInfo::Timezone. + # + # For example, those codes have the same effect. + # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) + # Time.zone = 'America/Los_Angeles' (TZInfo::Timezone) + # + # However, try_parse_cron only accepts the latter format. + # try_parse_cron('* * * * *', 'Pacific Time (US & Canada)') -> Doesn't work + # try_parse_cron('* * * * *', 'America/Los_Angeles') -> Works + # If you want to know more, please take a look + # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb + def try_parse_cron(cron, cron_timezone) + cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) + rescue + # noop + end + end + end +end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb new file mode 100644 index 00000000000..45fd0d4aa07 --- /dev/null +++ b/lib/gitlab/ci/status/build/action.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Action < Status::Extended + def label + if has_action? + @status.label + else + "#{@status.label} (not allowed)" + end + end + + def self.matches?(build, user) + build.action? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 67bbc3c4849..57b533bad99 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Cancelable < SimpleDelegator - include Status::Extended - + class Cancelable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 38ac6edc9f1..c852d607373 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -8,7 +8,8 @@ module Gitlab Status::Build::Retryable], [Status::Build::FailedAllowed, Status::Build::Play, - Status::Build::Stop]] + Status::Build::Stop], + [Status::Build::Action]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index 807afe24bd5..e42d3574357 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class FailedAllowed < SimpleDelegator - include Status::Extended - + class FailedAllowed < Status::Extended def label 'failed (allowed to fail)' end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 3495b8d0448..c6139f1b716 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Play < SimpleDelegator - include Status::Extended - + class Play < Status::Extended def label 'manual play action' end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 6b362af7634..505f80848b2 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Retryable < SimpleDelegator - include Status::Extended - + class Retryable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index e8530f2aaae..0b5199e5483 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Stop < SimpleDelegator - include Status::Extended - + class Stop < Status::Extended def label 'manual stop action' end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index d367c9bda69..1e8101f8949 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,13 +1,13 @@ module Gitlab module Ci module Status - module Extended - extend ActiveSupport::Concern + class Extended < SimpleDelegator + def initialize(status) + super(@status = status) + end - class_methods do - def matches?(_subject, _user) - raise NotImplementedError - end + def self.matches?(_subject, _user) + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb new file mode 100644 index 00000000000..cfd4329a923 --- /dev/null +++ b/lib/gitlab/ci/status/group/common.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Group + module Common + def has_details? + false + end + + def details_path + nil + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb new file mode 100644 index 00000000000..d118116cfc3 --- /dev/null +++ b/lib/gitlab/ci/status/group/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module Group + class Factory < Status::Factory + def self.common_helpers + Status::Group::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index a250c3fcb41..37dfe43fb62 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Pipeline - class Blocked < SimpleDelegator - include Status::Extended - + class Blocked < Status::Extended def text 'blocked' end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index d4cdab6957a..df6e76b0151 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -5,9 +5,7 @@ module Gitlab # Extended status used when pipeline or stage passed conditionally. # This means that failed jobs that are allowed to fail were present. # - class SuccessWarning < SimpleDelegator - include Status::Extended - + class SuccessWarning < Status::Extended def text 'passed' end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb new file mode 100644 index 00000000000..5b835bb669a --- /dev/null +++ b/lib/gitlab/ci/trace.rb @@ -0,0 +1,136 @@ +module Gitlab + module Ci + class Trace + attr_reader :job + + delegate :old_trace, to: :job + + def initialize(job) + @job = job + end + + def html(last_lines: nil) + read do |stream| + stream.html(last_lines: last_lines) + end + end + + def raw(last_lines: nil) + read do |stream| + stream.raw(last_lines: last_lines) + end + end + + def extract_coverage(regex) + read do |stream| + stream.extract_coverage(regex) + end + end + + def set(data) + write do |stream| + data = job.hide_secrets(data) + stream.set(data) + end + end + + def append(data, offset) + write do |stream| + current_length = stream.size + return -current_length unless current_length == offset + + data = job.hide_secrets(data) + stream.append(data, offset) + stream.size + end + end + + def exist? + current_path.present? || old_trace.present? + end + + def read + stream = Gitlab::Ci::Trace::Stream.new do + if current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + + def write + stream = Gitlab::Ci::Trace::Stream.new do + File.open(ensure_path, "a+b") + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + + def erase! + paths.each do |trace_path| + FileUtils.rm(trace_path, force: true) + end + + job.erase_old_trace! + end + + private + + def ensure_path + return current_path if current_path + + ensure_directory + default_path + end + + def ensure_directory + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) + end + end + + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + def paths + [ + default_path, + deprecated_path + ].compact + end + + def default_directory + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s + ) + end + + def default_path + File.join(default_directory, "#{job.id}.log") + end + + def deprecated_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project.ci_id.to_s, + "#{job.id}.log" + ) if job.project&.ci_id + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb new file mode 100644 index 00000000000..fa462cbe095 --- /dev/null +++ b/lib/gitlab/ci/trace/stream.rb @@ -0,0 +1,121 @@ +module Gitlab + module Ci + class Trace + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class Stream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 500.kilobytes + + attr_reader :stream + + delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + delegate :valid?, to: :stream, as: :present?, allow_nil: true + + def initialize + @stream = yield + @stream&.binmode + end + + def valid? + self.stream.present? + end + + def file? + self.path.present? + end + + def limit(last_bytes = LIMIT_SIZE) + if last_bytes < size + stream.seek(-last_bytes, IO::SEEK_END) + stream.readline + end + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def raw(last_lines: nil) + return unless valid? + + if last_lines.to_i > 0 + read_last_lines(last_lines) + else + stream.read + end.force_encoding(Encoding.default_external) + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html(last_lines: nil) + text = raw(last_lines: last_lines) + buffer = StringIO.new(text) + ::Ci::Ansi2html.convert(buffer).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex) + next unless matches.is_a?(Array) + next if matches.empty? + + match = matches.flatten.last + coverage = match.gsub(/\d+(\.\d+)?/).first + return coverage if coverage.present? + end + + nil + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + + private + + def read_last_lines(last_lines) + chunks = [] + pos = lines = 0 + max = stream.size + + # We want an extra line to make sure fist line has full contents + while lines <= last_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(last_lines).join + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb deleted file mode 100644 index 1d7ddeb3e0f..00000000000 --- a/lib/gitlab/ci/trace_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Ci - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 - class TraceReader - BUFFER_SIZE = 4096 - - attr_accessor :path, :buffer_size - - def initialize(new_path, buffer_size: BUFFER_SIZE) - self.path = new_path - self.buffer_size = Integer(buffer_size) - end - - def read(last_lines: nil) - if last_lines - read_last_lines(last_lines) - else - File.read(path) - end - end - - def read_last_lines(max_lines) - File.open(path) do |file| - chunks = [] - pos = lines = 0 - max = file.size - - # We want an extra line to make sure fist line has full contents - while lines <= max_lines && pos < max - pos += buffer_size - - buf = if pos <= max - file.seek(-pos, IO::SEEK_END) - file.read(buffer_size) - else # Reached the head, read only left - file.seek(0) - file.read(buffer_size - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) - end - - chunks.join.lines.last(max_lines).join - .force_encoding(Encoding.default_external) - end - end - end - end -end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 990b719ecfd..6e73361cad1 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -3,16 +3,33 @@ module Gitlab class FileCollection ConflictSideMissing = Class.new(StandardError) - attr_reader :merge_request, :our_commit, :their_commit + attr_reader :merge_request, :our_commit, :their_commit, :project - def initialize(merge_request) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit - end + delegate :repository, to: :project + + class << self + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def for_resolution(merge_request) + project = merge_request.source_project + + new(merge_request, project).tap do |file_collection| + project. + repository. + with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + + yield file_collection + end + end + end - def repository - merge_request.project.repository + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + def read_only(merge_request) + new(merge_request, merge_request.target_project) + end end def merge_index @@ -55,6 +72,15 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end + + private + + def initialize(merge_request, project) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + @project = project + end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 559e3939da6..cac31ea8cff 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -17,7 +17,7 @@ module Gitlab end def title - name.to_s.capitalize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def median diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 1e52b6614a1..5f9dc9a4303 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -13,12 +13,16 @@ module Gitlab :code end + def title + s_('CycleAnalyticsStage|Code') + end + def legend - "Related Merge Requests" + _("Related Merge Requests") end def description - "Time until first merge request" + _("Time until first merge request") end end end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 213994988a5..7b03811efb2 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -14,12 +14,16 @@ module Gitlab :issue end + def title + s_('CycleAnalyticsStage|Issue') + end + def legend - "Related Issues" + _("Related Issues") end def description - "Time before an issue gets scheduled" + _("Time before an issue gets scheduled") end end end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index bef3b95ff1b..1e11e84a9cb 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -7,7 +7,7 @@ module Gitlab test: :read_build, review: :read_merge_request, staging: :read_build, - production: :read_issue, + production: :read_issue }.freeze def self.get(*args) diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 45d51d30ccc..1a0afb56b4f 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -14,12 +14,16 @@ module Gitlab :plan end + def title + s_('CycleAnalyticsStage|Plan') + end + def legend - "Related Commits" + _("Related Commits") end def description - "Time before an issue starts implementation" + _("Time before an issue starts implementation") end end end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 9f387a02945..0fa8a65cb99 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -15,12 +15,16 @@ module Gitlab :production end + def title + s_('CycleAnalyticsStage|Production') + end + def legend - "Related Issues" + _("Related Issues") end def description - "From issue creation until deploy to production" + _("From issue creation until deploy to production") end def query diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index 4744be834de..cfbbdc43fd9 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -13,12 +13,16 @@ module Gitlab :review end + def title + s_('CycleAnalyticsStage|Review') + end + def legend - "Relative Merged Requests" + _("Related Merged Requests") end def description - "Time between merge request creation and merge/close" + _("Time between merge request creation and merge/close") end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index 3cdbe04fbaf..d5684bb9201 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -14,12 +14,16 @@ module Gitlab :staging end + def title + s_('CycleAnalyticsStage|Staging') + end + def legend - "Relative Deployed Builds" + _("Related Deployed Jobs") end def description - "From merge request merge until deploy to production" + _("From merge request merge until deploy to production") end end end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 43fa3795e5c..a917ddccac7 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -8,7 +8,7 @@ module Gitlab end def title - self.class.name.demodulize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def value diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 7b8faa4d854..bea78862757 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Commit < Base + def title + n_('Commit', 'Commits', value) + end + def value @value ||= count_commits end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 06032e9200e..099d798aac6 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Deploy < Base + def title + n_('Deploy', 'Deploys', value) + end + def value @value ||= @project.deployments.where("created_at > ?", @from).count end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 008468f24b9..9bbf7a2685f 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -9,7 +9,7 @@ module Gitlab end def title - 'New Issue' + n_('New Issue', 'New Issues', value) end def value diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index e96943833bc..2b5f72bef89 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -13,12 +13,16 @@ module Gitlab :test end + def title + s_('CycleAnalyticsStage|Test') + end + def legend - "Relative Builds Trigger by Commits" + _("Related Jobs") end def description - "Total test time for all commits/merges" + _("Total test time for all commits/merges") end def stage_query diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index f78106f5b10..8e74e18a311 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -36,7 +36,7 @@ module Gitlab user: { id: user.try(:id), name: user.try(:name), - email: user.try(:email), + email: user.try(:email) }, commit: { @@ -49,7 +49,7 @@ module Gitlab status: commit.status, duration: commit.duration, started_at: commit.started_at, - finished_at: commit.finished_at, + finished_at: commit.finished_at }, repository: { @@ -60,7 +60,7 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - }, + } } data diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index d76aa38f741..e81d19a7a2e 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -11,6 +11,7 @@ module Gitlab # ref: String, # user_id: String, # user_name: String, + # user_username: String, # user_email: String # project_id: String, # repository: { @@ -41,7 +42,7 @@ module Gitlab type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data - data = { + { object_kind: type, event_name: type, before: oldrev, @@ -51,6 +52,7 @@ module Gitlab message: message, user_id: user.id, user_name: user.name, + user_username: user.username, user_email: user.email, user_avatar: user.avatar_url, project_id: project.id, @@ -61,16 +63,15 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } - - data end # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - build(project, user, commits.last.id, commits.first.id, ref, commits) + commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + + build(project, user, commits.last&.id, commits.first&.id, ref, commits) end def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb new file mode 100644 index 00000000000..b42dc052949 --- /dev/null +++ b/lib/gitlab/data_builder/repository.rb @@ -0,0 +1,35 @@ +module Gitlab + module DataBuilder + module Repository + extend self + + # Produce a hash of post-receive data + def update(project, user, changes, refs) + { + event_name: 'repository_update', + + user_id: user.id, + user_name: user.name, + user_email: user.email, + user_avatar: user.avatar_url, + + project_id: project.id, + project: project.hook_attrs, + + changes: changes, + + refs: refs + } + end + + # Produce a hash of partial data for a single change + def single_change(oldrev, newrev, ref) + { + before: oldrev, + after: newrev, + ref: ref + } + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 63b8d0d3b9d..d0bd1299671 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -57,16 +57,16 @@ module Gitlab postgresql? ? "RANDOM()" : "RAND()" end - def true_value - if Gitlab::Database.postgresql? + def self.true_value + if postgresql? "'t'" else 1 end end - def false_value - if Gitlab::Database.postgresql? + def self.false_value + if postgresql? "'f'" else 0 diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fc445ab9483..e76c9abbe04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,6 +26,30 @@ module Gitlab add_index(table_name, column_name, options) end + # Removes an existed index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, :some_column + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ column: column_name })) + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When @@ -65,7 +89,8 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - ON DELETE #{on_delete} NOT VALID; + #{on_delete ? "ON DELETE #{on_delete}" : ''} + NOT VALID; EOF # Validate the existing constraint. This can potentially take a very @@ -90,6 +115,14 @@ module Gitlab execute('SET statement_timeout TO 0') if Database.postgresql? end + def true_value + Database.true_value + end + + def false_value + Database.false_value + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -226,6 +259,273 @@ module Gitlab raise error end end + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old - The old column name. + # new - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + def rename_column_concurrently(table, old, new, type: nil) + if transaction_open? + raise 'rename_column_concurrently can not be run inside a transaction' + end + + old_col = column_for(table, old) + new_type = type || old_col.type + + add_column(table, new, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new, old_col.default) if old_col.default + + trigger_name = rename_trigger_name(table, old, new) + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old) + quoted_new = quote_column_name(new) + + if Database.postgresql? + install_rename_triggers_for_postgresql(trigger_name, quoted_table, + quoted_old, quoted_new) + else + install_rename_triggers_for_mysql(trigger_name, quoted_table, + quoted_old, quoted_new) + end + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Changes the type of a column concurrently. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def change_column_type_concurrently(table, column, new_type) + temp_column = "#{column}_for_type_change" + + rename_column_concurrently(table, column, temp_column, type: new_type) + end + + # Performs cleanup of a concurrent type change. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def cleanup_concurrent_column_type_change(table, column) + temp_column = "#{column}_for_type_change" + + transaction do + # This has to be performed in a transaction as otherwise we might have + # inconsistent data. + cleanup_concurrent_column_rename(table, column, temp_column) + rename_column(table, temp_column, column) + end + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old - The name of the old column. + # new - The name of the new column. + def cleanup_concurrent_column_rename(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + if Database.postgresql? + remove_rename_triggers_for_postgresql(table, trigger_name) + else + remove_rename_triggers_for_mysql(trigger_name) + end + + remove_column(table, old) + end + + # Performs a concurrent column rename when using PostgreSQL. + def install_rename_triggers_for_postgresql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE OR REPLACE FUNCTION #{trigger}() + RETURNS trigger AS + $BODY$ + BEGIN + NEW.#{new} := NEW.#{old}; + RETURN NEW; + END; + $BODY$ + LANGUAGE 'plpgsql' + VOLATILE + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger} + BEFORE INSERT OR UPDATE + ON #{table} + FOR EACH ROW + EXECUTE PROCEDURE #{trigger}() + EOF + end + + # Installs the triggers necessary to perform a concurrent column rename on + # MySQL. + def install_rename_triggers_for_mysql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_insert + BEFORE INSERT + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_update + BEFORE UPDATE + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + end + + # Removes the triggers used for renaming a PostgreSQL column concurrently. + def remove_rename_triggers_for_postgresql(table, trigger) + execute("DROP TRIGGER #{trigger} ON #{table}") + execute("DROP FUNCTION #{trigger}()") + end + + # Removes the triggers used for renaming a MySQL column concurrently. + def remove_rename_triggers_for_mysql(trigger) + execute("DROP TRIGGER #{trigger}_insert") + execute("DROP TRIGGER #{trigger}_update") + end + + # Returns the (base) name to use for triggers when renaming columns. + def rename_trigger_name(table, old, new) + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) + end + + # Returns an Array containing the indexes for the given column + def indexes_for(table, column) + column = column.to_s + + indexes(table).select { |index| index.columns.include?(column) } + end + + # Returns an Array containing the foreign keys for the given column. + def foreign_keys_for(table, column) + column = column.to_s + + foreign_keys(table).select { |fk| fk.column == column } + end + + # Copies all indexes for the old column to a new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_indexes(table, old, new) + old = old.to_s + new = new.to_s + + indexes_for(table, old).each do |index| + new_columns = index.columns.map do |column| + column == old ? new : column + end + + # This is necessary as we can't properly rename indexes such as + # "ci_taggings_idx". + unless index.name.include?(old) + raise "The index #{index.name} can not be copied as it does not "\ + "mention the old column. You have to rename this index manually first." + end + + name = index.name.gsub(old, new) + + options = { + unique: index.unique, + name: name, + length: index.lengths, + order: index.orders + } + + # These options are not supported by MySQL, so we only add them if + # they were previously set. + options[:using] = index.using if index.using + options[:where] = index.where if index.where + + unless index.opclasses.blank? + opclasses = index.opclasses.dup + + # Copy the operator classes for the old column (if any) to the new + # column. + opclasses[new] = opclasses.delete(old) if opclasses[old] + + options[:opclasses] = opclasses + end + + add_concurrent_index(table, new_columns, options) + end + end + + # Copies all foreign keys for the old column to the new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_foreign_keys(table, old, new) + foreign_keys_for(table, old).each do |fk| + add_concurrent_foreign_key(fk.from_table, + fk.to_table, + column: new, + on_delete: fk.on_delete) + end + end + + # Returns the column for the given table and column name. + def column_for(table, name) + name = name.to_s + + columns(table).find { |column| column.name == name } + end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb new file mode 100644 index 00000000000..7ae5a4c17c8 --- /dev/null +++ b/lib/gitlab/database/multi_threaded_migration.rb @@ -0,0 +1,52 @@ +module Gitlab + module Database + module MultiThreadedMigration + MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection + + # This overwrites the default connection method so that every thread can + # use a thread-local connection, while still supporting all of Rails' + # migration methods. + def connection + Thread.current[MULTI_THREAD_AR_CONNECTION] || + ActiveRecord::Base.connection + end + + # Starts a thread-pool for N threads, along with N threads each using a + # single connection. The provided block is yielded from inside each + # thread. + # + # Example: + # + # with_multiple_threads(4) do + # execute('SELECT ...') + # end + # + # thread_count - The number of threads to start. + # + # join - When set to true this method will join the threads, blocking the + # caller until all threads have finished running. + # + # Returns an Array containing the started threads. + def with_multiple_threads(thread_count, join: true) + pool = Gitlab::Database.create_connection_pool(thread_count) + + threads = Array.new(thread_count) do + Thread.new do + pool.with_connection do |connection| + begin + Thread.current[MULTI_THREAD_AR_CONNECTION] = connection + yield + ensure + Thread.current[MULTI_THREAD_AR_CONNECTION] = nil + end + end + end + end + + threads.each(&:join) if join + + threads + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..5481024db8e --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,84 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + + def kind + type == 'Group' ? 'group' : 'user' + end + end + + class User < ActiveRecord::Base + self.table_name = 'users' + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..d60fd4bb551 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,132 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) + query.where(path_or_children) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..2958ad4b8e5 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,78 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + rename_user(old_full_path, new_full_path) if namespace.kind == 'user' + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def rename_user(old_username, new_username) + MigrationClasses::User.where(username: old_username) + .update_all(username: new_username) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb new file mode 100644 index 00000000000..c45ae8feb2c --- /dev/null +++ b/lib/gitlab/dependency_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + LINKERS = [ + GemfileLinker + ].freeze + + def self.linker(blob_name) + LINKERS.find { |linker| linker.support?(blob_name) } + end + + def self.link(blob_name, plain_text, highlighted_text) + linker = linker(blob_name) + return highlighted_text unless linker + + linker.link(plain_text, highlighted_text) + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb new file mode 100644 index 00000000000..5f4027e7e81 --- /dev/null +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -0,0 +1,109 @@ +module Gitlab + module DependencyLinker + class BaseLinker + class_attribute :file_type + + def self.support?(blob_name) + Gitlab::FileDetector.type_of(blob_name) == file_type + end + + def self.link(*args) + new(*args).link + end + + attr_accessor :plain_text, :highlighted_text + + def initialize(plain_text, highlighted_text) + @plain_text = plain_text + @highlighted_text = highlighted_text + end + + def link + link_dependencies + + highlighted_lines.join.html_safe + end + + private + + def package_url(name) + raise NotImplementedError + end + + def link_dependencies + raise NotImplementedError + end + + def package_link(name, url = package_url(name)) + return name unless url + + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + end + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call("gem") + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call("gem", "specific_package") + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call("github", /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call("name") + # # Will link `package` in `self.name = "package"` + def link_method_call(method_names, value = nil, &url_proc) + value = + case value + when String + Regexp.escape(value) + when nil + /[^'"]+/ + else + value + end + + method_names = Array(method_names).map { |name| Regexp.escape(name) } + + regex = %r{ + #{Regexp.union(method_names)} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + + # Links package names based on regex. + # + # Example: + # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) + # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` + def link_regex(regex) + highlighted_lines.map!.with_index do |rich_line, i| + marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + + marker.mark(regex, group: :name) do |text, left:, right:| + url = block_given? ? yield(text) : package_url(text) + package_link(text, url) + end + end + end + + def plain_lines + @plain_lines ||= plain_text.lines + end + + def highlighted_lines + @highlighted_lines ||= highlighted_text.lines + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb new file mode 100644 index 00000000000..9b82e126528 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -0,0 +1,29 @@ +module Gitlab + module DependencyLinker + class GemfileLinker < BaseLinker + self.file_type = :gemfile + + private + + def link_dependencies + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call("gem") + + # Link `github: "user/repo"` to https://github.com/user/repo + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| + "https://github.com/#{name}" + end + + # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + + # Link `source "https://rubygems.org"` to https://rubygems.org + link_method_call("source", %r{https?://[^'"]+}) { |url| url } + end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + end + end +end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 8406ca4269c..7948782aecc 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -18,6 +18,12 @@ module Gitlab head_sha == other.head_sha end + alias_method :eql?, :== + + def hash + [base_sha, start_sha, head_sha].hash + end + # There is only one case in which we will have `start_sha` and `head_sha`, # but not `base_sha`, which is when a diff is generated between an # orphaned branch and another branch, which means there _is_ no base, but diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb new file mode 100644 index 00000000000..c2a2eb15931 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -0,0 +1,17 @@ +module Gitlab + module Diff + class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + }.freeze + + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + symbol = MARKDOWN_SYMBOLS[mode] + "{#{symbol}#{text}#{symbol}}" + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 736933b1c4b..919965100ae 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,137 +1,21 @@ module Gitlab module Diff - class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - }.freeze - - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) - end - - def mark(line_inline_diffs, mode: nil, markdown: false) - return rich_line unless line_inline_diffs - - marker_ranges = [] - line_inline_diffs.each do |inline_diff_range| - # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range].flatten - # Turn the array of character positions into ranges - marker_ranges.concat(collapse_ranges(inline_diff_positions)) - end - - offset = 0 - - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "<span class='#{html_class_names(marker_ranges, mode, index)}'>" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "</span>" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + class InlineDiffMarker < Gitlab::StringRangeMarker + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - - rich_line.html_safe end private - def html_class_names(marker_ranges, mode, index) + def html_class_names(left, right, mode) class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 + class_names << "left" if left + class_names << "right" if right class_names << mode if mode class_names.join(" ") end - - # Mapping of character positions in the raw line, to the rich (highlighted) line - def position_mapping - @position_mapping ||= begin - mapping = [] - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - rich_char = rich_line[rich_pos] - - # The raw and rich lines are the same except for HTML tags, - # so skip over any `<...>` segment - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - # multi-char HTML entities in the rich line correspond to a single character in the raw line - if rich_char == '&' - multichar_mapping = [rich_pos] - until rich_char == ';' - rich_pos += 1 - multichar_mapping << rich_pos - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = multichar_mapping - else - mapping[raw_pos] = rich_pos - end - - rich_pos += 1 - end - - mapping - end - end - - # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - # Inserts tags around the characters identified by the given range - def insert_around_range(text, range, before, after, offset = 0) - # Just to be sure - return offset if offset + range.end + 1 > text.length - - text.insert(offset + range.begin, before) - offset += before.length - - text.insert(offset + range.end + 1, after) - offset += after.length - - offset - end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 114656958e3..0a15c6d9358 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -33,6 +33,10 @@ module Gitlab new_pos unless removed? || meta? end + def line + new_line || old_line + end + def unchanged? type.nil? end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..e89ff238ec7 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -16,7 +16,7 @@ module Gitlab end def trace(old_position) - return unless old_diff_refs.complete? && new_diff_refs.complete? + return unless old_diff_refs&.complete? && new_diff_refs&.complete? return unless old_position.diff_refs == old_diff_refs # Suppose we have an MR with source branch `feature` and target branch `master`. @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 496ee0bdcb0..38e27513281 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -131,10 +131,12 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", %W[git apply --check --3way #{patch_path}] ) do |output, status| + puts output unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') @@ -310,6 +312,17 @@ module Gitlab Resolve them, stage the changes and commit them. + If the patch couldn't be applied cleanly, use the following command: + + # In the EE repo + $ git apply --reject path/to/#{ce_branch}.patch + + This option makes git apply the parts of the patch that are applicable, + and leave the rejected hunks in corresponding `.rej` files. + You can then resolve the conflicts highlighted in `.rej` by + manually applying the correct diff from the `.rej` file to the file with conflicts. + When finished, you can delete the `.rej` files and commit your changes. + ⚠️ Don't forget to push your branch to gitlab-ee: # In the EE repo diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 32cece8316b..83440ae227d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -21,7 +21,7 @@ module Gitlab content_type: attachment.content_type } - link = ::Projects::UploadService.new(project, file).execute + link = UploadService.new(project, file).execute attachments << link if link ensure tmp.close! diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index 35ea2e0ef59..b07c68d1498 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -5,7 +5,11 @@ require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze + HANDLERS = [ + UnsubscribeHandler, + CreateNoteHandler, + CreateIssueHandler + ].freeze def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..a616a80e8f5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project&.full_path) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index d87ba427f4b..31579e94a87 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' @@ -8,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -27,32 +28,22 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project&.full_path) + end + private def author sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end def create_note - Notes::CreateService.new( - project, - author, - note: message, - noteable_type: sent_notification.noteable_type, - noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code, - position: sent_notification.position, - type: sent_notification.note_type - ).execute + sent_notification.create_reply(message) end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..5894384da5d 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end @@ -17,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project&.full_path) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..0d6b08b5d29 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,6 +31,8 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) + handler.execute end @@ -56,9 +57,8 @@ module Gitlab end def key_from_additional_headers(mail) - references = ensure_references_array(mail.references) - - find_key_from_references(references) + find_key_from_references(mail) || + find_key_from_delivered_to_header(mail) end def ensure_references_array(references) @@ -69,15 +69,24 @@ module Gitlab # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app Gitlab::IncomingEmail.scan_fallback_references(references) + when nil + [] end end - def find_key_from_references(references) - references.find do |mail_id| + def find_key_from_references(mail) + ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end + + def find_key_from_delivered_to_header(mail) + Array(mail[:delivered_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index a16d9fc2265..e3e36b35ce9 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -54,7 +54,7 @@ module Gitlab unicode_version: emoji_unicode_version(emoji_name) } - ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], data: data) + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data) end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index ffbc6e17dc5..270d67dd50c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -1,27 +1,23 @@ module Gitlab module EtagCaching class Middleware - RESERVED_WORDS = ProjectPathValidator::RESERVED.map { |word| "/#{word}/" }.join('|') - ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) - ) - def initialize(app) @app = app end def call(env) - return @app.call(env) unless enabled_for_current_route?(env) - Gitlab::Metrics.add_event(:etag_caching_middleware_used) + route = Gitlab::EtagCaching::Router.match(env) + return @app.call(env) unless route + + track_event(:etag_caching_middleware_used, route) etag, cached_value_present = get_etag(env) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag - Gitlab::Metrics.add_event(:etag_caching_cache_hit) - [304, { 'ETag' => etag }, ['']] + handle_cache_hit(etag, route) else - track_cache_miss(if_none_match, cached_value_present) + track_cache_miss(if_none_match, cached_value_present, route) status, headers, body = @app.call(env) headers['ETag'] = etag @@ -31,10 +27,6 @@ module Gitlab private - def enabled_for_current_route?(env) - ROUTE_REGEXP.match(env['PATH_INFO']) - end - def get_etag(env) cache_key = env['PATH_INFO'] store = Gitlab::EtagCaching::Store.new @@ -52,15 +44,27 @@ module Gitlab %Q{W/"#{value}"} end - def track_cache_miss(if_none_match, cached_value_present) + def handle_cache_hit(etag, route) + track_event(:etag_caching_cache_hit, route) + + status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 + + [status_code, { 'ETag' => etag }, []] + end + + def track_cache_miss(if_none_match, cached_value_present, route) if if_none_match.blank? - Gitlab::Metrics.add_event(:etag_caching_header_missing) + track_event(:etag_caching_header_missing, route) elsif !cached_value_present - Gitlab::Metrics.add_event(:etag_caching_key_not_found) + track_event(:etag_caching_key_not_found, route) else - Gitlab::Metrics.add_event(:etag_caching_resource_changed) + track_event(:etag_caching_resource_changed, route) end end + + def track_event(name, route) + Gitlab::Metrics.add_event(name, endpoint: route.name) + end end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb new file mode 100644 index 00000000000..ba31041d0c1 --- /dev/null +++ b/lib/gitlab/etag_caching/router.rb @@ -0,0 +1,51 @@ +module Gitlab + module EtagCaching + class Router + Route = Struct.new(:regexp, :name) + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) + ROUTES = [ + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), + 'issue_notes' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/realtime_changes\z), + 'issue_title' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), + 'commit_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), + 'new_merge_request_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), + 'merge_request_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), + 'project_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), + 'project_pipeline' + ) + ].freeze + + def self.match(env) + ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + end + end + end +end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index c9ca4cadd1c..a8cb7fc3fe7 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -5,15 +5,33 @@ module Gitlab # a README or a CONTRIBUTING file. module FileDetector PATTERNS = { + # Project files readme: /\Areadme/i, changelog: /\A(changelog|history|changes|news)/i, license: /\A(licen[sc]e|copying)(\..+|\z)/i, contributing: /\Acontributing/i, version: 'version', + avatar: /\Alogo\.(png|jpg|gif)\z/, + + # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - avatar: /\Alogo\.(png|jpg|gif)\z/ + route_map: 'route-map.yml', + + # Dependency files + cartfile: /\ACartfile/, + composer_json: 'composer.json', + gemfile: /\A(Gemfile|gems\.rb)\z/, + gemfile_lock: 'Gemfile.lock', + gemspec: /\.gemspec\z/, + godeps_json: 'Godeps.json', + package_json: 'package.json', + podfile: 'Podfile', + podspec_json: /\.podspec\.json\z/, + podspec: /\.podspec\z/, + requirements_txt: /requirements\.txt\z/, + yarn_lock: 'yarn.lock' }.freeze # Returns an Array of file types based on the given paths. diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb new file mode 100644 index 00000000000..093d9ed8092 --- /dev/null +++ b/lib/gitlab/file_finder.rb @@ -0,0 +1,32 @@ +# This class finds files in a repository by name and content +# the result is joined and sorted by file name +module Gitlab + class FileFinder + BATCH_SIZE = 100 + + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + end + + def find(query) + blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = Gitlab::ProjectSearchResults.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| + results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 222bcdcbf9c..3dcee681c72 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -122,15 +122,15 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - iid: bug['ixBug'], - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed', - created_at: date, - updated_at: DateTime.parse(bug['dtLastUpdated']) + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_ids: [assignee_id], + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d3df3f1bca1..936606152e9 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -4,6 +4,8 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze + CommandError = Class.new(StandardError) + class << self def ref_name(ref) ref.sub(/\Arefs\/(tags|heads)\//, '') diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e56eb0d3beb..c1b31618e0d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -8,7 +8,7 @@ module Gitlab # the user. We load as much as we can for encoding detection # (Linguist) and LFS pointer parsing. All other cases where we need full # blob data should use load_all_data!. - MAX_DATA_DISPLAY_SIZE = 10485760 + MAX_DATA_DISPLAY_SIZE = 10.megabytes attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary @@ -90,7 +90,7 @@ module Gitlab name: blob_entry[:name], data: '', path: path, - commit_id: sha, + commit_id: sha ) end end @@ -109,10 +109,6 @@ module Gitlab @binary.nil? ? super : @binary == true end - def empty? - !data || data == '' - end - def data encode! @data end @@ -132,6 +128,10 @@ module Gitlab encode! @name end + def truncated? + size && (size > loaded_size) + end + # Valid LFS object pointer is a text file consisting of # version # oid @@ -153,16 +153,20 @@ module Gitlab def lfs_size if has_lfs_version_key? size = data.match(/(?<=size )([0-9]+)/) - return size[1] if size + return size[1].to_i if size end nil end - def truncated? - size && (size > loaded_size) + def external_storage + return unless lfs_pointer? + + :lfs end + alias_method :external_size, :lfs_size + private def has_lfs_version_key? diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 586380da94a..124526e4b59 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,40 @@ module Gitlab module Git class Branch < Ref + def initialize(repository, name, target) + if target.is_a?(Gitaly::FindLocalBranchResponse) + target = target_from_gitaly_local_branches_response(target) + end + + super(repository, name, target) + end + + def target_from_gitaly_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name, + author_email: response.commit_author.email, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name, + committer_email: response.commit_committer.email + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 3a73697dc5d..297531db4cc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -19,13 +19,7 @@ module Gitlab def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) - methods = [:message, :parent_ids, :authored_date, :author_name, - :author_email, :committed_date, :committer_name, - :committer_email] - - methods.all? do |method| - send(method) == other.send(method) - end + id && id == other.id end class << self @@ -55,6 +49,7 @@ module Gitlab # Commit.find(repo, 'master') # def find(repo, commit_id = "HEAD") + return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) obj = if commit_id.is_a?(String) @@ -192,6 +187,10 @@ module Gitlab Commit.diff_from_parent(raw_commit, options) end + def deltas + @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + end + def has_zero_stats? stats.total.zero? rescue diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 019be151353..31d1b66b4f7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -183,6 +183,8 @@ module Gitlab when Gitaly::CommitDiffResponse init_from_gitaly(raw_diff) prune_diff_if_eligible(collapse) + when Gitaly::CommitDelta + init_from_gitaly(raw_diff) when nil raise "Nil as raw diff passed" else @@ -278,15 +280,15 @@ module Gitlab end end - def init_from_gitaly(diff_msg) - @diff = diff_msg.raw_chunks.join - @new_path = encode!(diff_msg.to_path.dup) - @old_path = encode!(diff_msg.from_path.dup) - @a_mode = diff_msg.old_mode.to_s(8) - @b_mode = diff_msg.new_mode.to_s(8) - @new_file = diff_msg.from_id == BLANK_SHA - @renamed_file = diff_msg.from_path != diff_msg.to_path - @deleted_file = diff_msg.to_id == BLANK_SHA + def init_from_gitaly(msg) + @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) + @new_path = encode!(msg.to_path.dup) + @old_path = encode!(msg.from_path.dup) + @a_mode = msg.old_mode.to_s(8) + @b_mode = msg.new_mode.to_s(8) + @new_file = msg.from_id == BLANK_SHA + @renamed_file = msg.from_path != msg.to_path + @deleted_file = msg.to_id == BLANK_SHA end def prune_diff_if_eligible(collapse = false) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 4e45ec7c174..bcbad8ec829 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -15,7 +15,6 @@ module Gitlab @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file @all_diffs = !!options.fetch(:all_diffs, false) @no_collapse = !!options.fetch(:no_collapse, true) - @deltas_only = !!options.fetch(:deltas_only, false) @line_count = 0 @byte_count = 0 @@ -27,8 +26,6 @@ module Gitlab if @populated # @iterator.each is slower than just iterating the array in place @array.each(&block) - elsif @deltas_only - each_delta(&block) else Gitlab::GitalyClient.migrate(:commit_raw_diffs) do each_patch(&block) @@ -81,14 +78,6 @@ module Gitlab files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes end - def each_delta - @iterator.each_delta.with_index do |delta, i| - diff = Gitlab::Git::Diff.new(delta) - - yield @array[i] = diff - end - end - def each_patch @iterator.each_with_index do |raw, i| # First yield cached Diff instances from @array diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb new file mode 100644 index 00000000000..0fdc57ec954 --- /dev/null +++ b/lib/gitlab/git/env.rb @@ -0,0 +1,38 @@ +module Gitlab + module Git + # Ephemeral (per request) storage for environment variables that some Git + # commands may need. + # + # For example, in pre-receive hooks, new objects are put in a temporary + # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved + # (this would break push rules for instance). + # + # This class is thread-safe via RequestStore. + class Env + WHITELISTED_GIT_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY + GIT_ALTERNATE_OBJECT_DIRECTORIES + ].freeze + + def self.set(env) + return unless RequestStore.active? + + RequestStore.store[:gitlab_git_env] = whitelist_git_env(env) + end + + def self.all + return {} unless RequestStore.active? + + RequestStore.fetch(:gitlab_git_env) { {} } + end + + def self.[](key) + all[key] + end + + def self.whitelist_git_env(env) + env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access + end + end + end +end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index af1744c9c46..1add037fa5f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,8 +1,12 @@ module Gitlab module Git class Index + IndexError = Class.new(StandardError) + DEFAULT_MODE = 0o100644 + ACTIONS = %w(create create_dir update move delete).freeze + attr_reader :repository, :raw_index def initialize(repository) @@ -23,9 +27,8 @@ module Gitlab def create(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end add_blob(options) @@ -34,13 +37,12 @@ module Gitlab def create_dir(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end if dir_exists?(options[:file_path]) - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + raise IndexError, "A directory with this name already exists" end options = options.dup @@ -55,7 +57,7 @@ module Gitlab file_entry = get(options[:file_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" end add_blob(options, mode: file_entry[:mode]) @@ -66,7 +68,11 @@ module Gitlab file_entry = get(options[:previous_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" + end + + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end raw_index.remove(options[:previous_path]) @@ -77,9 +83,8 @@ module Gitlab def delete(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + unless get(options[:file_path]) + raise IndexError, "A file with this name doesn't exist" end raw_index.remove(options[:file_path]) @@ -95,10 +100,20 @@ module Gitlab end def normalize_path(path) + unless path + raise IndexError, "You must provide a file path" + end + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - if pathname.each_filename.include?('..') - raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + pathname.each_filename do |segment| + if segment == '..' + raise IndexError, 'Path cannot include directory traversal' + end + + unless segment =~ Gitlab::Regex.file_name_regex + raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" + end end pathname.to_s @@ -106,6 +121,10 @@ module Gitlab def add_blob(options, mode: nil) content = options[:content] + unless content + raise IndexError, "You must provide content" + end + content = Base64.decode64(content) if options[:encoding] == 'base64' detect = CharlockHolmes::EncodingDetector.new.detect(content) @@ -119,7 +138,7 @@ module Gitlab raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) rescue Rugged::IndexError => e - raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + raise IndexError, e.message end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4e72519c81d..b9f1ac144b6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,10 @@ module Gitlab class Repository include Gitlab::Git::Popen + ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY + GIT_ALTERNATE_OBJECT_DIRECTORIES + ].freeze SEARCH_CONTEXT_LINES = 3 NoRepository = Class.new(StandardError) @@ -23,11 +27,17 @@ module Gitlab # Rugged repo object attr_reader :rugged + attr_reader :storage + # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(path) - @path = path - @name = path.split("/").last + def initialize(storage, relative_path) + @storage = storage + @relative_path = relative_path + + storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @path = File.join(storage_path, @relative_path) + @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) end @@ -37,7 +47,13 @@ module Gitlab # Default branch in the repository def root_ref - @root_ref ||= discover_default_branch + @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| + if is_enabled + gitaly_ref_client.default_branch_name + else + discover_default_branch + end + end end # Alias to old method for compatibility @@ -46,7 +62,7 @@ module Gitlab end def rugged - @rugged ||= Rugged::Repository.new(path) + @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories) rescue Rugged::RepositoryError, Rugged::OSError raise NoRepository.new('no repository for such path') end @@ -54,18 +70,26 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - branches.map(&:name) + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names + else + branches.map(&:name) + end + end end # Returns an Array of Branches - def branches - rugged.branches.map do |rugged_ref| + def branches(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| begin Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) rescue Rugged::ReferenceError # Omit invalid branch end - end.compact.sort_by(&:name) + end.compact + + sort_branches(branches, sort_by) end def reload_rugged @@ -86,28 +110,57 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref end - def local_branches - rugged.branches.each(:local).map do |branch| - Gitlab::Git::Branch.new(self, branch.name, branch.target) + def local_branches(sort_by: nil) + gitaly_migrate(:local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch| + Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch) + end + else + branches(filter: :local, sort_by: sort_by) + end end end # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end # Returns an Array of tag names def tag_names - rugged.tags.map { |t| t.name } + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names + else + rugged.tags.map { |t| t.name } + end + end end # Returns an Array of Tags @@ -215,7 +268,7 @@ module Gitlab 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(prefix, storage_path, format), - 'CommitId' => commit.id, + 'CommitId' => commit.id } end @@ -411,6 +464,11 @@ module Gitlab rugged.merge_base(from, to) end + # Returns true is +from+ is direct ancestor to +to+, otherwise false + def is_ancestor?(from, to) + gitaly_commit_client.is_ancestor(from, to) + end + # Return an array of Diff objects that represent the diff # between +from+ and +to+. See Diff::filter_diff_options for the allowed # diff options. The +options+ hash can also include :break_rewrites to @@ -419,6 +477,23 @@ module Gitlab Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) end + # Returns a RefName for a given SHA + def ref_name_for_sha(ref_path, sha) + raise ArgumentError, "sha can't be empty" unless sha.present? + + gitaly_migrate(:find_ref_name) do |is_enabled| + if is_enabled + gitaly_ref_client.find_ref_name(sha, ref_path) + else + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, @path).first.split.last + end + end + end + # Returns commits collection # # Ex. @@ -434,7 +509,10 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :date(default) or :topo + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) actual_options = options.dup @@ -462,11 +540,8 @@ module Gitlab end end - if actual_options[:order] == :topo - walker.sorting(Rugged::SORT_TOPO) - else - walker.sorting(Rugged::SORT_NONE) - end + sort_type = rugged_sort_type(actual_options[:order]) + walker.sorting(sort_type) commits = [] offset = actual_options[:skip] @@ -812,27 +887,6 @@ module Gitlab rugged.remotes[remote_name].push(refspecs) end - # Merge the +source_name+ branch into the +target_name+ branch. This is - # equivalent to `git merge --no_ff +source_name+`, since a merge commit - # is always created. - def merge(source_name, target_name, options = {}) - our_commit = rugged.branches[target_name].target - their_commit = rugged.branches[source_name].target - - raise "Invalid merge target" if our_commit.nil? - raise "Invalid merge source" if their_commit.nil? - - merge_index = rugged.merge_commits(our_commit, their_commit) - return false if merge_index.conflicts? - - actual_options = options.merge( - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - update_ref: "refs/heads/#{target_name}" - ) - Rugged::Commit.create(rugged, actual_options) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -920,8 +974,16 @@ module Gitlab @attributes.attributes(path) end + def gitaly_repository + Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + end + private + def alternate_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact + end + # Get the content of a blob for a given commit. If the blob is a commit # (for submodules) then return the blob's OID. def blob_content(commit, blob_name) @@ -1056,56 +1118,6 @@ module Gitlab end end - def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) - git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) - - # Put files into a directory before archiving - prefix = "#{archive_name(treeish)}/" - git_archive_cmd << "--prefix=#{prefix}" - - # Format defaults to tar - git_archive_cmd << "--format=#{format}" if format - - git_archive_cmd += %W(-- #{treeish}) - - open(filename, 'w') do |file| - # Create a pipe to act as the '|' in 'git archive ... | gzip' - pipe_rd, pipe_wr = IO.pipe - - # Get the compression process ready to accept data from the read end - # of the pipe - compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) - # The read end belongs to the compression process now; we should - # close our file descriptor for it. - pipe_rd.close - - # Start 'git archive' and tell it to write into the write end of the - # pipe. - git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) - # The write end belongs to 'git archive' now; close it. - pipe_wr.close - - # When 'git archive' and the compression process are finished, we are - # done. - Process.waitpid(git_archive_pid) - raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? - Process.waitpid(compress_pid) - raise "#{compress_cmd.join(' ')} failed" unless $?.success? - end - end - - def nice(cmd) - nice_cmd = %w(nice -n 20) - unless unsupported_platform? - nice_cmd += %w(ionice -c 2 -n 7) - end - nice_cmd + cmd - end - - def unsupported_platform? - %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? - end - # Returns true if the index entry has the special file mode that denotes # a submodule. def submodule?(index_entry) @@ -1197,6 +1209,53 @@ module Gitlab diff.find_similar!(break_rewrites: break_rewrites) diff.each_patch end + + def sort_branches(branches, sort_by) + case sort_by + when 'name' + branches.sort_by(&:name) + when 'updated_desc' + branches.sort do |a, b| + b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date + end + when 'updated_asc' + branches.sort do |a, b| + a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date + end + else + branches + end + end + + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE | Rugged::SORT_TOPO + } + + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 79dd0cf7df2..a16b0ed76f4 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,41 +1,42 @@ module Gitlab module Git class RevList - attr_reader :project, :env - - ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze - - def initialize(oldrev, newrev, project:, env: nil) - @project = project - @env = env.presence || {} - @args = [Gitlab.config.git.bin_path, - "--git-dir=#{project.repository.path_to_repo}", - "rev-list", - "--max-count=1", - oldrev, - "^#{newrev}"] + attr_reader :oldrev, :newrev, :path_to_repo + + def initialize(path_to_repo:, newrev:, oldrev: nil) + @oldrev = oldrev + @newrev = newrev + @path_to_repo = path_to_repo end - def execute - Gitlab::Popen.popen(@args, nil, parse_environment_variables) + # This method returns an array of new references + def new_refs + execute([*base_args, newrev, '--not', '--all']) end - def valid? - environment_variables.all? do |(name, value)| - value.to_s.start_with?(project.repository.path_to_repo) - end + # This methods returns an array of missed references + def missed_ref + execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end private - def parse_environment_variables - return {} unless valid? + def execute(args) + output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + + unless status.zero? + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + end - environment_variables + output.split("\n") end - def environment_variables - @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + def base_args + [ + Gitlab.config.git.bin_path, + "--git-dir=#{path_to_repo}", + 'rev-list' + ] end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index f7450e8b58f..d41256d9a84 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -33,9 +33,9 @@ module Gitlab root_id: root_tree.oid, name: entry[:name], type: entry[:type], - mode: entry[:filemode], + mode: entry[:filemode].to_s(8), path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha, + commit_id: sha ) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index eea2f206902..99724db8da2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -18,13 +18,12 @@ module Gitlab attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:, env: {}) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) - @env = env end def check(cmd, changes) @@ -152,7 +151,6 @@ module Gitlab change, user_access: user_access, project: project, - env: @env, skip_authorization: deploy_key?, protocol: protocol ).exec diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 6babea144c7..742118b76a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -1,43 +1,30 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :repo_path, :identifier, :changes, :project + attr_reader :project, :identifier, :changes - def initialize(repo_path, identifier, changes) - repo_path.gsub!(/\.git\z/, '') - repo_path.gsub!(/\A\//, '') - - @repo_path = repo_path + def initialize(project, identifier, changes) + @project = project @identifier = identifier @changes = deserialize_changes(changes) - - retrieve_project_and_type - end - - def wiki? - @type == :wiki - end - - def regular_project? - @type == :project end def identify(revision) super(identifier, project, revision) end - private + def changes_refs + return enum_for(:changes_refs) unless block_given? - def retrieve_project_and_type - @type = :project - @project = Project.find_by_full_path(@repo_path) + changes.each do |change| + oldrev, newrev, ref = change.strip.split(' ') - if @repo_path.end_with?('.wiki') && !@project - @type = :wiki - @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) + yield oldrev, newrev, ref end end + private + def deserialize_changes(changes) changes = utf8_encode_changes(changes) changes.lines diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a0dbe0a8c11..72466700c05 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,26 +4,42 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - def self.configure_channel(storage, address) - @addresses ||= {} - @addresses[storage] = address - @channels ||= {} - @channels[storage] = new_channel(address) - end + MUTEX = Mutex.new + private_constant :MUTEX - def self.new_channel(address) - # NOTE: Gitaly currently runs on a Unix socket, so permissions are - # handled using the file system and no additional authentication is - # required (therefore the :this_channel_is_insecure flag) - GRPC::Core::Channel.new(address, {}, :this_channel_is_insecure) + def self.stub(name, storage) + MUTEX.synchronize do + @stubs ||= {} + @stubs[storage] ||= {} + @stubs[storage][name] ||= begin + klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + addr = address(storage) + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + klass.new(addr, :this_channel_is_insecure) + end + end end - def self.get_channel(storage) - @channels[storage] + def self.clear_stubs! + MUTEX.synchronize do + @stubs = nil + end end - def self.get_address(storage) - @addresses[storage] + def self.address(storage) + params = Gitlab.config.repositories.storages[storage] + raise "storage not found: #{storage.inspect}" if params.nil? + + address = params['gitaly_address'] + unless address.present? + raise "storage #{storage.inspect} is missing a gitaly_address" + end + + unless URI(address).scheme.in?(%w(tcp unix)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + end + + address end def self.enabled? diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 9c714a3ee45..4491903d788 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,23 +5,55 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - class << self - def diff_from_parent(commit, options = {}) - project = commit.project - channel = GitalyClient.get_channel(project.repository_storage) - stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel) - repo = Gitaly::Repository.new(path: project.repository.path_to_repo) - parent = commit.parents[0] - parent_id = parent ? parent.id : EMPTY_TREE_ID - request = Gitaly::CommitDiffRequest.new( - repository: repo, - left_commit_id: parent_id, - right_commit_id: commit.id - ) - - Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + stub = GitalyClient.stub(:commit, @repository.storage) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + stub.commit_is_ancestor(request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) + Gitlab::Git::DiffCollection.new(response, options) + end + + def commit_deltas(commit) + request_params = commit_diff_request_params(commit) + + response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def diff_service_stub + GitalyClient.stub(:diff, @repository.storage) + end end end end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb index cbfb129c002..719554eac52 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -3,18 +3,14 @@ module Gitlab class Notifications attr_accessor :stub - def initialize(repo_path) - full_path = Gitlab::RepoPath.strip_storage_path(repo_path). - sub(/\.git\z/, '').sub(/\.wiki\z/, '') - @project = Project.find_by_full_path(full_path) - - channel = GitalyClient.get_channel(@project.repository_storage) - @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel) + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = GitalyClient.stub(:notifications, repository.storage) end def post_receive - repository = Gitaly::Repository.new(path: @project.repository.path_to_repo) - request = Gitaly::PostReceiveRequest.new(repository: repository) + request = Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) @stub.post_receive(request) end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb new file mode 100644 index 00000000000..227fe45642e --- /dev/null +++ b/lib/gitlab/gitaly_client/ref.rb @@ -0,0 +1,72 @@ +module Gitlab + module GitalyClient + class Ref + attr_accessor :stub + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = GitalyClient.stub(:ref, repository.storage) + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + branch_name = stub.find_default_branch_name(request).name + + Gitlab::Git.branch_name(branch_name) + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/') + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) + consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') + end + + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @gitaly_repo, + commit_id: commit_id, + prefix: ref_prefix + ) + + stub.find_ref_name(request).name + end + + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + consume_branches_response(stub.find_local_branches(request)) + end + + private + + def consume_refs_response(response, prefix:) + response.flat_map do |r| + r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } + end + end + + def sort_by_param(sort_by) + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map { |r| r.branches } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb new file mode 100644 index 00000000000..86d055d3533 --- /dev/null +++ b/lib/gitlab/gitaly_client/util.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + module Util + class << self + def repository(repository_storage, relative_path) + Gitaly::Repository.new( + path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), + storage_name: repository_storage, + relative_path: relative_path + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 5d29e698b27..8aa885fb811 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -11,6 +11,14 @@ module Gitlab sha.present? && ref.present? end + def user + raw_data.user&.login || 'unknown' + end + + def short_sha + Commit.truncate_sha(sha) + end + private def branch_exists? diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index eea4a91f17d..a8c0b47e786 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -157,7 +157,7 @@ module Gitlab end def restore_source_branch(pull_request) - project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name) + project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end def restore_target_branch(pull_request) diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 6f5ac4dac0d..977cd0423ba 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -10,7 +10,7 @@ module Gitlab description: description, state: state, author_id: author_id, - assignee_id: assignee_id, + assignee_ids: Array(assignee_id), created_at: raw_data.created_at, updated_at: raw_data.updated_at } diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index add7236e339..150afa31432 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,8 +1,8 @@ module Gitlab module GithubImport class PullRequestFormatter < IssuableFormatter - delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true + delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true def attributes { @@ -20,7 +20,8 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: raw_data.updated_at + updated_at: raw_data.updated_at, + imported: true } end @@ -37,13 +38,20 @@ module Gitlab end def source_branch_name - @source_branch_name ||= begin - if cross_project? - "pull/#{number}/#{source_branch_repo.full_name}/#{source_branch_ref}" + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed else - source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}" + source_branch_ref end - end + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}" + end + + def source_branch_exists? + !cross_project? && source_branch.exists? end def target_branch @@ -51,13 +59,17 @@ module Gitlab end def target_branch_name - @target_branch_name ||= begin - target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}" - end + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}" end def cross_project? - source_branch.repo.id != target_branch.repo.id + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id end def opened? diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb new file mode 100644 index 00000000000..07c0abcce23 --- /dev/null +++ b/lib/gitlab/gl_repository.rb @@ -0,0 +1,20 @@ +module Gitlab + module GlRepository + def self.gl_repository(project, is_wiki) + "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" + end + + def self.parse(gl_repository) + match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) + unless match_data + raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" + end + + type, id = match_data.captures + project = Project.find_by(id: id) + wiki = type == 'wiki' + + [project, wiki] + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5ab84266b7d..6200bd460ea 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -10,11 +10,14 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') + gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.gitlab_url = Gitlab.config.gitlab.url if current_user gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name + gon.current_user_avatar_url = current_user.avatar_url end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..1b43440673c 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -92,13 +108,13 @@ module Gitlab end issue = Issue.create!( - iid: raw_issue['id'], - project_id: project.id, - title: raw_issue['title'], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_ids: [assignee_id], + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb new file mode 100644 index 00000000000..7de6d4d9367 --- /dev/null +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -0,0 +1,45 @@ +module Gitlab + module HealthChecks + module BaseAbstractCheck + def name + super.demodulize.underscore + end + + def human_name + name.sub(/_check$/, '').capitalize + end + + def readiness + raise NotImplementedError + end + + def liveness + HealthChecks::Result.new(true) + end + + def metrics + [] + end + + protected + + def metric(name, value, **labels) + Metric.new(name, value, labels) + end + + def with_timing(proc) + start = Time.now + result = proc.call + yield result, Time.now.to_f - start.to_f + end + + def catch_timeout(seconds, &block) + begin + Timeout.timeout(seconds.to_i, &block) + rescue Timeout::Error => ex + ex + end + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb new file mode 100644 index 00000000000..fd94984f8a2 --- /dev/null +++ b/lib/gitlab/health_checks/db_check.rb @@ -0,0 +1,29 @@ +module Gitlab + module HealthChecks + class DbCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'db_ping' + end + + def is_successful?(result) + result == '1' + end + + def check + catch_timeout 10.seconds do + if Gitlab::Database.postgresql? + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + else + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb new file mode 100644 index 00000000000..df962d203b7 --- /dev/null +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -0,0 +1,117 @@ +module Gitlab + module HealthChecks + class FsShardsCheck + extend BaseAbstractCheck + + class << self + def readiness + repository_storages.map do |storage_name| + begin + tmp_file_path = tmp_file_path(storage_name) + + if !storage_stat_test(storage_name) + HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) + elsif !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + rescue RuntimeError => ex + message = "unexpected error #{ex} when checking storage #{storage_name}" + Rails.logger.error(message) + HealthChecks::Result.new(false, message, shard: storage_name) + ensure + delete_test_file(tmp_file_path) + end + end + end + + def metrics + repository_storages.flat_map do |storage_name| + tmp_file_path = tmp_file_path(storage_name) + [ + operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), + operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), + operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + ].flatten + end + end + + private + + RANDOM_STRING = SecureRandom.hex(1000).freeze + + def operation_metrics(ok_metric, latency_metric, operation, **labels) + with_timing operation do |result, elapsed| + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] + end + rescue RuntimeError => ex + Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + [metric(ok_metric, 0, **labels)] + end + + def repository_storages + @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages + end + + def storages_paths + @storage_paths ||= Gitlab.config.repositories.storages + end + + def with_timeout(args) + %w{timeout 1}.concat(args) + end + + def tmp_file_path(storage_name) + Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + end + + def path(storage_name) + storages_paths&.dig(storage_name, 'path') + end + + def storage_stat_test(storage_name) + stat_path = File.join(path(storage_name), '.') + begin + _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} })) + status == 0 + rescue Errno::ENOENT + File.exist?(stat_path) && File::Stat.new(stat_path).readable? + end + end + + def storage_write_test(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin| + stdin.write(RANDOM_STRING) + end + status == 0 + rescue Errno::ENOENT + written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT + written_bytes == RANDOM_STRING.length + end + + def storage_read_test(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin| + stdin.write(RANDOM_STRING) + end + status == 0 + rescue Errno::ENOENT + file_contents = File.read(tmp_path) rescue Errno::ENOENT + file_contents == RANDOM_STRING + end + + def delete_test_file(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} })) + status == 0 + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + end + end + end +end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb new file mode 100644 index 00000000000..1a2eab0b005 --- /dev/null +++ b/lib/gitlab/health_checks/metric.rb @@ -0,0 +1,3 @@ +module Gitlab::HealthChecks + Metric = Struct.new(:name, :value, :labels) +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb new file mode 100644 index 00000000000..57bbe5b3ad0 --- /dev/null +++ b/lib/gitlab/health_checks/redis_check.rb @@ -0,0 +1,25 @@ +module Gitlab + module HealthChecks + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis.with(&:ping) + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb new file mode 100644 index 00000000000..8086760023e --- /dev/null +++ b/lib/gitlab/health_checks/result.rb @@ -0,0 +1,3 @@ +module Gitlab::HealthChecks + Result = Struct.new(:success, :message, :labels) +end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb new file mode 100644 index 00000000000..fbe1645c1b1 --- /dev/null +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -0,0 +1,43 @@ +module Gitlab + module HealthChecks + module SimpleAbstractCheck + include BaseAbstractCheck + + def readiness + check_result = check + if is_successful?(check_result) + HealthChecks::Result.new(true) + elsif check_result.is_a?(Timeout::Error) + HealthChecks::Result.new(false, "#{human_name} check timed out") + else + HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}") + end + end + + def metrics + with_timing method(:check) do |result, elapsed| + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency", elapsed) + ] + end + end + + private + + def metric_prefix + raise NotImplementedError + end + + def is_successful?(result) + raise NotImplementedError + end + + def check + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d787d5db4a0..83bc230df3e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -13,6 +13,8 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end + attr_reader :blob_name + def initialize(blob_name, blob_content, repository: nil) @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @@ -21,16 +23,9 @@ module Gitlab end def highlight(text, continue: true, plain: false) - if plain - hl_lexer = Rouge::Lexers::PlainText - continue = false - else - hl_lexer = self.lexer - end - - @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe - rescue - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + highlighted_text = highlight_text(text, continue: continue, plain: plain) + highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text end def lexer @@ -50,5 +45,27 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end + + def highlight_text(text, continue: true, plain: false) + if plain + highlight_plain(text) + else + highlight_rich(text, continue: continue) + end + end + + def highlight_plain(text) + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + def highlight_rich(text, continue: true) + @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + rescue + highlight_plain(text) + end + + def link_dependencies(text, highlighted_text) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb new file mode 100644 index 00000000000..3411516319f --- /dev/null +++ b/lib/gitlab/i18n.rb @@ -0,0 +1,26 @@ +module Gitlab + module I18n + extend self + + AVAILABLE_LANGUAGES = { + 'en' => 'English', + 'es' => 'Español', + 'de' => 'Deutsch' + }.freeze + + def available_locales + AVAILABLE_LANGUAGES.keys + end + + def set_locale(current_user) + requested_locale = current_user&.preferred_language || ::I18n.default_locale + locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = locale + end + + def reset_locale + FastGettext.set_locale(::I18n.default_locale) + ::I18n.locale = ::I18n.default_locale + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8b327cfc226..27d5a9198b6 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.6'.freeze + VERSION = '0.1.7'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb new file mode 100644 index 00000000000..d4adeeb3797 --- /dev/null +++ b/lib/gitlab/import_export/hash_util.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class HashUtil + def self.deep_symbolize_array!(array) + return if array.blank? + + array.map! do |hash| + hash.deep_symbolize_keys! + + yield(hash) if block_given? + + hash + end + end + + def self.deep_symbolize_array_with_date!(array) + self.deep_symbolize_array!(array) do |hash| + hash.select { |k, _v| k.to_s.end_with?('_date') }.each do |key, value| + hash[key] = Time.zone.parse(value) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ab74c8782f6..d0f3cf2b514 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -40,20 +40,18 @@ project_tree: - :events - :statuses - :triggers - - :deploy_keys + - :pipeline_schedules - :services - :hooks - protected_branches: - :merge_access_levels - :push_access_levels + - protected_tags: + - :create_access_levels - :project_feature # Only include the following attributes for the models specified. included_attributes: - project: - - :description - - :visibility_level - - :archived user: - :id - :email @@ -63,6 +61,32 @@ included_attributes: # Do not include the following attributes for the models specified. excluded_attributes: + project: + - :name + - :path + - :namespace_id + - :creator_id + - :import_url + - :import_status + - :avatar + - :import_type + - :import_source + - :import_error + - :mirror + - :runners_token + - :repository_storage + - :repository_read_only + - :lfs_enabled + - :import_jid + - :created_at + - :updated_at + - :import_jid + - :import_jid + - :id + - :star_count + - :last_activity_at + - :last_repository_updated_at + - :last_repository_check_at snippets: - :expired_at merge_request_diff: @@ -84,8 +108,11 @@ methods: - :type statuses: - :type - - :gl_project_id services: - :type merge_request_diff: - :utf8_st_diffs + merge_requests: + - :diff_head_sha + project: + - :description_html diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 063ce74ecad..fbdd74788bc 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab end def execute - if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb new file mode 100644 index 00000000000..c20adc20bfd --- /dev/null +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -0,0 +1,41 @@ +module Gitlab + module ImportExport + class MergeRequestParser + FORKED_PROJECT_ID = -1 + + def initialize(project, diff_head_sha, merge_request, relation_hash) + @project = project + @diff_head_sha = diff_head_sha + @merge_request = merge_request + @relation_hash = relation_hash + end + + def parse! + if fork_merge_request? && @diff_head_sha + @merge_request.source_project_id = @relation_hash['project_id'] + + fetch_ref unless branch_exists?(@merge_request.source_branch) + create_target_branch unless branch_exists?(@merge_request.target_branch) + end + + @merge_request + end + + def create_target_branch + @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha) + end + + def fetch_ref + @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch) + end + + def branch_exists?(branch_name) + @project.repository.branch_exists?(branch_name) + end + + def fork_merge_request? + @relation_hash['source_project_id'] == FORKED_PROJECT_ID + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index cda6ddf0443..84ab1977dfa 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -52,7 +52,11 @@ module Gitlab create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + relation_hash_list = @tree_hash[relation_key.to_s] + + next unless relation_hash_list + + relation_hash = create_relation(relation_key, relation_hash_list) saved << restored_project.append_or_update_attribute(relation_key, relation_hash) end saved.all? @@ -67,14 +71,14 @@ module Gitlab def restore_project return @project unless @tree_hash - @project.update(project_params) + @project.update_columns(project_params) @project end def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations - value.is_a?(Array) || key == key.singularize + value.respond_to?(:each) && !Project.column_names.include?(key) end end @@ -119,7 +123,7 @@ module Gitlab relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, user: @user, - project_id: restored_project.id) + project: restored_project) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index a1e7159fe42..eb7f5120592 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -15,7 +15,10 @@ module Gitlab # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + attributes = @attributes_finder.find(:project) + project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} + + project_attributes.merge(include: build_hash(@tree)) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d44563333a5..19e23a4715f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -5,15 +5,17 @@ module Gitlab pipelines: 'Ci::Pipeline', statuses: 'commit_status', triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -29,11 +31,12 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id) @members_mapper = members_mapper @user = user + @project = project @imported_object_retries = 0 end @@ -66,7 +69,7 @@ module Gitlab remove_encrypted_attributes! @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] - set_st_diffs if @relation_name == :merge_request_diff + set_st_diff_commits if @relation_name == :merge_request_diff end def update_user_references @@ -105,6 +108,8 @@ module Gitlab imported_object do |object| object.commit_id = nil end + elsif @relation_name == :merge_requests + MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else imported_object end @@ -115,7 +120,7 @@ module Gitlab # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1 + @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID end # project_id may not be part of the export, but we always need to populate it if required. @@ -166,6 +171,7 @@ module Gitlab def imported_object yield(existing_or_new_object) if block_given? existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + existing_or_new_object rescue ActiveRecord::RecordNotUnique # as the operation is not atomic, retry in the unlikely scenario an INSERT is @@ -180,7 +186,7 @@ module Gitlab end def admin_user? - @user.is_admin? + @user.admin? end def parsed_relation_hash @@ -188,8 +194,11 @@ module Gitlab relation_class: relation_class) end - def set_st_diffs + def set_st_diff_commits @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') + + HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) + HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end def existing_or_new_object diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 3a7af363548..4a6091488c8 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -38,7 +38,7 @@ module Gitlab url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], headers: Hash.new { |h, k| h[k] = [] }, - created_at: created_at, + created_at: created_at } end end @@ -64,7 +64,7 @@ module Gitlab tty: true, stdin: true, stdout: true, - stderr: true, + stderr: true }.to_query + '&' + EXEC_COMMAND case url.scheme diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 28129198438..6fdf68641e2 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -39,7 +39,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption, + encryption: encryption ) opts.merge!(auth_options) if has_auth? @@ -124,9 +124,9 @@ module Gitlab def name_proc if allow_username_or_email_login - Proc.new { |name| name.gsub(/@.*\z/, '') } + proc { |name| name.gsub(/@.*\z/, '') } else - Proc.new { |name| name } + proc { |name| name } end end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..49285e35251 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,6 +1,11 @@ module Gitlab module MarkupHelper - module_function + extend self + + MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze + ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze + OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -8,10 +13,7 @@ module Gitlab # # Returns boolean def markup?(filename) - gitlab_markdown?(filename) || - asciidoc?(filename) || - filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki - .mediawiki .rst)) + EXTENSIONS.include?(extension(filename)) end # Public: Determines if a given filename is compatible with @@ -21,7 +23,7 @@ module Gitlab # # Returns boolean def gitlab_markdown?(filename) - filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown)) + MARKDOWN_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename has AsciiDoc extension. @@ -30,7 +32,7 @@ module Gitlab # # Returns boolean def asciidoc?(filename) - filename.downcase.end_with?(*%w(.adoc .ad .asciidoc)) + ASCIIDOC_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename is plain text. @@ -39,12 +41,17 @@ module Gitlab # # Returns boolean def plain?(filename) - filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + extension(filename) == 'txt' || filename.casecmp('readme').zero? end def previewable?(filename) markup?(filename) end + + private + + def extension(filename) + File.extname(filename).downcase.delete('.') + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..cb8db2f1e9f 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -49,6 +49,9 @@ module Gitlab end end end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) end def self.prepare_metrics(metrics) @@ -138,6 +141,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +157,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f98481c6d3a..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::CreateService.new(nil, user_params).build + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index e67acf28c94..31a24460f0f 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -4,21 +4,13 @@ module Gitlab # Public: Converts the provided markup into HTML. # # input - the source text in a markup format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref # def self.render(file_name, input, context) html = GitHub::Markup.render(file_name, input). force_encoding(input.encoding) + context[:pipeline] = :markup - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb new file mode 100644 index 00000000000..f0c50584f07 --- /dev/null +++ b/lib/gitlab/polling_interval.rb @@ -0,0 +1,22 @@ +module Gitlab + class PollingInterval + include Gitlab::CurrentSettings + + HEADER_NAME = 'Poll-Interval'.freeze + + def self.set_header(response, interval:) + if polling_enabled? + multiplier = current_application_settings.polling_interval_multiplier + value = (interval * multiplier).to_i + else + value = -1 + end + + response.headers[HEADER_NAME] = value.to_s + end + + def self.polling_enabled? + !current_application_settings.polling_interval_multiplier.zero? + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index db325c00705..561aa9e162c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -62,7 +62,7 @@ module Gitlab data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') end - OpenStruct.new( + FoundBlob.new( filename: filename, basename: basename, ref: ref, @@ -82,26 +82,14 @@ module Gitlab private def blobs - @blobs ||= begin - blobs = project.repository.search_files_by_content(query, repository_ref).first(100) - found_file_names = Set.new - - results = blobs.map do |blob| - blob = self.class.parse_search_result(blob) - found_file_names << blob.filename - - [blob.filename, blob] - end - - project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| - results << [filename, nil] unless found_file_names.include?(filename) - end + return [] unless Ability.allowed?(@current_user, :download_code, @project) - results.sort_by(&:first) - end + @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) end def wiki_blobs + return [] unless Ability.allowed?(@current_user, :read_wiki, @project) + @wiki_blobs ||= begin if project.wiki_enabled? && query.present? project_wiki = ProjectWiki.new(project) diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb new file mode 100644 index 00000000000..2a2eb4ae57f --- /dev/null +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -0,0 +1,26 @@ +module Gitlab + module Prometheus + module Queries + class BaseQuery + attr_accessor :client + delegate :query_range, :query, to: :client, prefix: true + + def raw_memory_usage_query(environment_slug) + %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} + end + + def raw_cpu_usage_query(environment_slug) + %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} + end + + def initialize(client) + @client = client + end + + def query(*args) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb new file mode 100644 index 00000000000..2cc08731f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -0,0 +1,26 @@ +module Gitlab::Prometheus::Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug + + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), + + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb new file mode 100644 index 00000000000..01d756d7284 --- /dev/null +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -0,0 +1,20 @@ +module Gitlab::Prometheus::Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus_client.rb index 62239779454..5b51a1779dd 100644 --- a/lib/gitlab/prometheus.rb +++ b/lib/gitlab/prometheus_client.rb @@ -2,7 +2,7 @@ module Gitlab PrometheusError = Class.new(StandardError) # Helper methods to interact with Prometheus network services & resources - class Prometheus + class PrometheusClient attr_reader :api_url def initialize(api_url:) @@ -13,18 +13,18 @@ module Gitlab json_api_get('query', query: '1') end - def query(query) + def query(query, time: Time.now) get_result('vector') do - json_api_get('query', query: query) + json_api_get('query', query: query, time: time.to_f) end end - def query_range(query, start: 8.hours.ago) + def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', query: query, start: start.to_f, - end: Time.now.utc.to_f, + end: stop.to_f, step: 1.minute.to_i) end end @@ -50,6 +50,12 @@ module Gitlab def get(url) handle_response(HTTParty.get(url)) + rescue SocketError + raise PrometheusError, "Can't connect to #{url}" + rescue OpenSSL::SSL::SSLError + raise PrometheusError, "#{url} contains invalid SSL data" + rescue HTTParty::Error + raise PrometheusError, "Network connection error" end def handle_response(response) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 5e5f5ff1589..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end @@ -73,22 +77,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def file_path_regex - @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze - end - - def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." - end - - def directory_traversal_regex - @directory_traversal_regex ||= /\.{2}/.freeze - end - - def directory_traversal_regex_message - "cannot include directory traversal." - end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze @@ -121,6 +109,13 @@ module Gitlab git_reference_regex end + ## + # Docker Distribution Registry 2.4.1 repository name rules + # + def container_repository_name_regex + @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + end + def environment_name_regex @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 4b1d828c45c..878e03f61d7 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -2,18 +2,29 @@ module Gitlab module RepoPath NotFoundError = Class.new(StandardError) - def self.strip_storage_path(repo_path) - result = nil + def self.parse(repo_path) + project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) + project = Project.find_by_full_path(project_path) + if project_path.end_with?('.wiki') && !project + project = Project.find_by_full_path(project_path.chomp('.wiki')) + wiki = true + else + wiki = false + end + + [project, wiki] + end + + def self.strip_storage_path(repo_path, fail_on_not_found: true) + result = repo_path - Gitlab.config.repositories.storages.values.each do |params| - storage_path = params['path'] - if repo_path.start_with?(storage_path) - result = repo_path.sub(storage_path, '') - break - end + storage = Gitlab.config.repositories.storages.values.find do |params| + repo_path.start_with?(params['path']) end - if result.nil? + if storage + result = result.sub(storage['path'], '') + elsif fail_on_not_found raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index ccfa517e04b..efe8095beea 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,5 +1,26 @@ module Gitlab class SearchResults + class FoundBlob + attr_reader :id, :filename, :basename, :ref, :startline, :data + + def initialize(opts = {}) + @id = opts.fetch(:id, nil) + @filename = opts.fetch(:filename, nil) + @basename = opts.fetch(:basename, nil) + @ref = opts.fetch(:ref, nil) + @startline = opts.fetch(:startline, nil) + @data = opts.fetch(:data, nil) + end + + def path + filename + end + + def no_highlighting? + false + end + end + attr_reader :current_user, :query # Limit search results by passed projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 117fc508135..2442c2ded3b 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -11,7 +11,7 @@ module Gitlab Raven.user_context( id: current_user.id, email: current_user.email, - username: current_user.username, + username: current_user.username ) end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index da8d8ddb8ed..b1d6ea665b7 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -35,7 +35,7 @@ module Gitlab end def strip_key(key) - key.split(/ /)[0, 2].join(' ') + key.split(/[ ]+/)[0, 2].join(' ') end private @@ -83,7 +83,27 @@ module Gitlab # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, '800']) + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) + raise Error, output unless status.zero? + true + end + + # Fetch remote for repository + # + # name - project path with namespace + # remote - remote name + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # + # Ex. + # fetch_remote("gitlab/gitlab-ci", "upstream") + # + def fetch_remote(storage, name, remote, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] + args << '--force' if forced + args << '--no-tags' if no_tags + + output, status = Popen.popen(args) raise Error, output unless status.zero? true end @@ -174,7 +194,10 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - FileUtils.mkdir_p(full_path(storage, name), mode: 0770) unless exists?(storage, name) + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + rescue Errno::EEXIST => e + Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") end # Remove directory from repositories storage diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 11e5f1b645c..ca8d3271541 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -72,6 +72,8 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. + # true = job is still running + # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } @@ -82,6 +84,17 @@ module Gitlab end end + # Returns the JIDs that are completed + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of completed JIDs + def self.completed_jids(job_ids) + Sidekiq.redis do |redis| + job_ids.reject { |jid| redis.exists(key_for(jid)) } + end + end + def self.key_for(jid) STATUS_KEY % jid end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index d47609f490d..00983b3284a 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -2,7 +2,9 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - Gitlab::SidekiqStatus.set(job['jid']) + status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + + Gitlab::SidekiqStatus.set(job['jid'], status_expiration) yield end end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb index 60d35be2599..12a385f90fd 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -1,16 +1,19 @@ module Gitlab module SlashCommands class CommandDefinition - attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + attr_accessor :name, :aliases, :description, :explanation, :params, + :condition_block, :parse_params_block, :action_block def initialize(name, attributes = {}) @name = name - @aliases = attributes[:aliases] || [] - @description = attributes[:description] || '' - @params = attributes[:params] || [] + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @explanation = attributes[:explanation] || '' + @params = attributes[:params] || [] @condition_block = attributes[:condition_block] - @action_block = attributes[:action_block] + @parse_params_block = attributes[:parse_params_block] + @action_block = attributes[:action_block] end def all_names @@ -28,14 +31,20 @@ module Gitlab context.instance_exec(&condition_block) end + def explain(context, opts, arg) + return unless available?(opts) + + if explanation.respond_to?(:call) + execute_block(explanation, context, arg) + else + explanation + end + end + def execute(context, opts, arg) return if noop? || !available?(opts) - if arg.present? - context.instance_exec(arg, &action_block) - elsif action_block.arity == 0 - context.instance_exec(&action_block) - end + execute_block(action_block, context, arg) end def to_h(opts) @@ -52,6 +61,23 @@ module Gitlab params: params } end + + private + + def execute_block(block, context, arg) + if arg.present? + parsed = parse_params(arg, context) + context.instance_exec(parsed, &block) + elsif block.arity == 0 + context.instance_exec(&block) + end + end + + def parse_params(arg, context) + return arg unless parse_params_block + + context.instance_exec(arg, &parse_params_block) + end end end end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb index 50b0937d267..614bafbe1b2 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/slash_commands/dsl.rb @@ -44,6 +44,22 @@ module Gitlab @params = params end + # Allows to give an explanation of what the command will do when + # executed. This explanation is shown when rendering the Markdown + # preview. + # + # Example: + # + # explanation do |arguments| + # "Adds label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def explanation(text = '', &block) + @explanation = block_given? ? block : text + end + # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # It accepts a block that will be evaluated with the context given to @@ -61,6 +77,24 @@ module Gitlab @condition_block = block end + # Allows to perform initial parsing of parameters. The result is passed + # both to `command` and `explanation` blocks, instead of the raw + # parameters. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # parse_params do |raw| + # raw.strip + # end + # command :command_key do |parsed| + # # Awesome code block + # end + def parse_params(&block) + @parse_params_block = block + end + # Registers a new command which is recognizeable from body of email or # comment. # It accepts aliases and takes a block. @@ -75,11 +109,13 @@ module Gitlab definition = CommandDefinition.new( name, - aliases: aliases, - description: @description, - params: @params, - condition_block: @condition_block, - action_block: block + aliases: aliases, + description: @description, + explanation: @explanation, + params: @params, + condition_block: @condition_block, + parse_params_block: @parse_params_block, + action_block: block ) self.command_definitions << definition @@ -89,8 +125,14 @@ module Gitlab end @description = nil + @explanation = nil @params = nil @condition_block = nil + @parse_params_block = nil + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] end end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb new file mode 100644 index 00000000000..94fba0a221a --- /dev/null +++ b/lib/gitlab/string_range_marker.rb @@ -0,0 +1,102 @@ +module Gitlab + class StringRangeMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(marker_ranges) + return rich_line unless marker_ranges + + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + + offset = 0 + # Mark each range + rich_marker_ranges.each_with_index do |range, i| + offset_range = (range.begin + offset)..(range.end + offset) + original_text = rich_line[offset_range] + + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + + rich_line[offset_range] = text + + offset += text.length - original_text.length + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + end +end diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb new file mode 100644 index 00000000000..7ebf1c0428c --- /dev/null +++ b/lib/gitlab/string_regex_marker.rb @@ -0,0 +1,13 @@ +module Gitlab + class StringRegexMarker < StringRangeMarker + def mark(regex, group: 0, &block) + regex_match = raw_line.match(regex) + return rich_line unless regex_match + + begin_index, end_index = regex_match.offset(group) + name_range = begin_index..(end_index - 1) + + super([name_range], &block) + end + end +end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..bcba2e3e1b6 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,66 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + ci_pipeline_schedules: ::Ci::PipelineSchedule.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index f260c0c535f..3b922da7ced 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -28,16 +28,33 @@ module Gitlab true end + def can_create_tag?(ref) + return false unless can_access_git? + + if ProtectedTag.protected?(project, ref) + project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + else + user.can?(:push_code, project) + end + end + + def can_delete_branch?(ref) + return false unless can_access_git? + + if ProtectedBranch.protected?(project, ref) + user.can?(:delete_protected_branch, project) + else + user.can?(:push_code, project) + end + end + def can_push_to_branch?(ref) return false unless can_access_git? - if project.protected_branch?(ref) + if ProtectedBranch.protected?(project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - has_access = access_levels.any? { |access_level| access_level.check_access(user) } - - has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) else user.can?(:push_code, project) end @@ -46,9 +63,8 @@ module Gitlab def can_merge_to_branch?(ref) return false unless can_access_git? - if project.protected_branch?(ref) - access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + if ProtectedBranch.protected?(project, ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) else user.can?(:push_code, project) end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 8f1d1fdc02e..2e31f4462f9 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -63,7 +63,7 @@ module Gitlab end def allowed_for?(user, level) - user.is_admin? || allowed_level?(level.to_i) + user.admin? || allowed_level?(level.to_i) end # Return true if the specified level is allowed for the current user. diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 6fe85af3c30..fe37e4da94f 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -16,15 +16,32 @@ module Gitlab SECRET_LENGTH = 32 class << self - def git_http_ok(repository, user) + def git_http_ok(repository, is_wiki, user, action) + project = repository.project + repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), - RepoPath: repository.path_to_repo, + GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + RepoPath: repo_path } if Gitlab.config.gitaly.enabled - address = Gitlab::GitalyClient.get_address(repository.project.repository_storage) - params[:GitalySocketPath] = URI(address).path + address = Gitlab::GitalyClient.address(project.repository_storage) + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 + false + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + + params[:GitalyAddress] = address if feature_enabled end params @@ -34,7 +51,7 @@ module Gitlab { StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", LfsOid: oid, - LfsSize: size, + LfsSize: size } end @@ -45,7 +62,7 @@ module Gitlab def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id, + 'BlobId' => blob.id } [ @@ -110,7 +127,7 @@ module Gitlab 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], 'Header' => terminal[:headers], - 'MaxSessionTime' => terminal[:max_session_time], + 'MaxSessionTime' => terminal[:max_session_time] } } details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) @@ -148,12 +165,12 @@ module Gitlab encoded_message, secret, true, - { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' } ) end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb new file mode 100644 index 00000000000..d2c420efdaf --- /dev/null +++ b/lib/microsoft_teams/activity.rb @@ -0,0 +1,19 @@ +module MicrosoftTeams + class Activity + def initialize(title:, subtitle:, text:, image:) + @title = title + @subtitle = subtitle + @text = text + @image = image + end + + def prepare + { + 'activityTitle' => @title, + 'activitySubtitle' => @subtitle, + 'activityText' => @text, + 'activityImage' => @image + } + end + end +end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb new file mode 100644 index 00000000000..3bef68a1bcb --- /dev/null +++ b/lib/microsoft_teams/notifier.rb @@ -0,0 +1,46 @@ +module MicrosoftTeams + class Notifier + def initialize(webhook) + @webhook = webhook + @header = { 'Content-type' => 'application/json' } + end + + def ping(options = {}) + result = false + + begin + response = HTTParty.post( + @webhook.to_str, + headers: @header, + body: body(options) + ) + + result = true if response + rescue HTTParty::Error, StandardError => error + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + end + + result + end + + private + + def body(options = {}) + result = { 'sections' => [] } + + result['title'] = options[:title] + result['summary'] = options[:pretext] + result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare + + attachments = options[:attachments] + unless attachments.blank? + result['sections'] << { + 'title' => 'Details', + 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] + } + end + + result.to_json + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 09e121e5120..6e351365de0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -326,8 +326,7 @@ start_gitlab() { echo "Gitaly is already running with pid $gapid, not restarting" else $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ - $app_root/bin/with_env $gitaly_dir/env \ - $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 & fi fi diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 2301ec9b228..99b3168d9eb 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 5293f5af12d..87ca39b079b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -19,8 +19,9 @@ namespace :gemojione do entry = { category: emoji_hash['category'], moji: emoji_hash['moji'], + description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), - digest: hash_digest, + digest: hash_digest } resultant_emoji_map[name] = entry diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake new file mode 100644 index 00000000000..0aa21a4bd13 --- /dev/null +++ b/lib/tasks/gettext.rake @@ -0,0 +1,14 @@ +require "gettext_i18n_rails/tasks" + +namespace :gettext do + # Customize list of translatable files + # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files + def files_to_translate + folders = %W(app lib config #{locale_path}).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + + Dir.glob( + "{#{folders}}/**/*.{#{exts}}" + ) + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a9a48f7188f..f41c73154f5 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,8 +431,7 @@ namespace :gitlab do def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" + puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| repo_base_path = repository_storage['path'] @@ -443,15 +442,16 @@ namespace :gitlab do break end - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + user_id = uid_for(gitlab_shell_ssh_user) + root_group_id = gid_for('root') + group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] + if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) puts "yes".color(:green) else puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" ) for_more_information( see_installation_guide_section "GitLab Shell" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 5476438b8fa..139ab70e125 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -65,6 +65,7 @@ namespace :gitlab do migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines .map { |file| Rails.root.join(file.strip).to_s } .select { |file| File.file?(file) } + .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index c288e17ac8d..3c5bc0146a1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,23 +1,74 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| + require 'toml' + warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') - tag = "v#{Gitlab::GitalyClient.expected_server_version}" - repo = 'https://gitlab.com/gitlab-org/gitaly.git' + version = Gitlab::GitalyClient.expected_server_version - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do + create_gitaly_configuration run_command!([command]) end end + + desc "GitLab | Print storage configuration in TOML format" + task storage_config: :environment do + require 'toml' + + puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" + puts "# This is in TOML format suitable for use in Gitaly's config.toml file." + + config = Gitlab.config.repositories.storages.map do |key, val| + { name: key, path: val['path'] } + end + + puts TOML.dump(storage: config) + end + + private + + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only write a config.toml file in the most common and simplest case: the + # case where we have exactly one Gitaly process and we are sure it is + # running locally because it uses a Unix socket. + def create_gitaly_configuration + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + File.open("config.toml", "w") do |f| + f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + end + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index a2a2db487b7..e3883278886 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -16,6 +16,8 @@ namespace :gitlab do redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a # check Git version git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a + # check Go version + go_version = run_and_match(%w(go version), /go version (.+)/).to_a puts "" puts "System information".color(:yellow) @@ -30,6 +32,7 @@ namespace :gitlab do puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" + puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd2fda54e62..ee2cdcdea1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,19 +1,18 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:tag, :repo] => :environment do |t, args| + task :install, [:repo] => :environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = "v#{default_version}" - args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do @@ -42,8 +41,14 @@ namespace :gitlab do # Generate config.yml based on existing gitlab settings File.open("config.yml", "w+") {|f| f.puts config.to_yaml} - # Launch installation process - system(*%w(bin/install) + repository_storage_paths_args) + [ + %w(bin/install) + repository_storage_paths_args, + %w(bin/compile) + ].each do |cmd| + unless Kernel.system(*cmd) + raise "command failed: #{cmd.join(' ')}" + end + end end # (Re)create hooks diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index cdba2262bc2..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -147,41 +147,30 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - checkout_tag(tag, target_dir) - else - clone_repo(repo, target_dir) - end + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end - reset_to_tag(tag, target_dir) + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end - def checkout_tag(tag, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) end - def reset_to_tag(tag_wanted, target_dir) - tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end + def reset_to_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index dbdfb335a5c..59c32bbe7a4 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,7 +5,7 @@ namespace :gitlab do end def update(template) - sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) unless clone_repository(template.repo_url, dir) @@ -44,7 +44,11 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ + ), + Template.new( + "https://gitlab.com/gitlab-org/Dockerfile.git", + /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) ].freeze diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index baea94bf8ca..e7ac0b5859f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,16 +1,16 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') - tag = "v#{Gitlab::Workhorse.version}" - repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + version = Gitlab::Workhorse.version - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 00000000000..bc76d7edc55 --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,142 @@ +require 'benchmark' +require 'rainbow/ext/string' + +class GithubImport + def self.run!(*args) + new(*args).run! + end + + def initialize(token, gitlab_username, project_path, extras) + @options = { url: 'https://api.github.com', token: token, verbose: true } + @project_path = project_path + @current_user = User.find_by_username(gitlab_username) + @github_repo = extras.empty? ? nil : extras.first + end + + def run! + @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! + + raise 'No repo found!' unless @repo + + show_warning! + + @project = Project.find_by_full_path(@project_path) || new_project + + import! + end + + private + + def show_warning! + puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts "Permission checks are ignored. Press any key to continue.".color(:red) + + STDIN.getch + + puts 'Starting the import (this could take a while)'.color(:green) + end + + def import! + @project.import_start + + timings = Benchmark.measure do + Github::Import.new(@project, @options).execute + end + + puts "Import finished. Timings: #{timings}".color(:green) + + @project.import_finish + end + + def new_project + Project.transaction do + namespace_path, _sep, name = @project_path.rpartition('/') + namespace = find_or_create_namespace(namespace_path) + + Projects::CreateService.new( + @current_user, + name: name, + path: name, + description: @repo['description'], + namespace_id: namespace.id, + visibility_level: visibility_level, + import_type: 'github', + import_source: @repo['full_name'], + skip_wiki: @repo['has_wiki'] + ).execute + end + end + + def find_or_create_namespace(names) + return @current_user.namespace if names == @current_user.namespace_path + return @current_user.namespace unless @current_user.can_create_group? + + full_path_namespace = Namespace.find_by_full_path(names) + + return full_path_namespace if full_path_namespace + + names.split('/').inject(nil) do |parent, name| + begin + namespace = Group.create!(name: name, + path: name, + owner: @current_user, + parent: parent) + namespace.add_owner(@current_user) + + namespace + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + Namespace.where(parent: parent).find_by_path_or_name(name) + end + end + end + + def full_path_namespace(names) + @full_path_namespace ||= Namespace.find_by_full_path(names) + end + + def visibility_level + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end +end + +class GithubRepos + def initialize(options, current_user, github_repo) + @options = options + @current_user = current_user + @github_repo = github_repo + end + + def choose_one! + return found_github_repo if @github_repo + + repos.each do |repo| + print "ID: #{repo['id'].to_s.bright}".color(:green) + print "\tName: #{repo['full_name']}\n".color(:green) + end + + print 'ID? '.bright + + repos.find { |repo| repo['id'] == repo_id } + end + + def found_github_repo + repos.find { |repo| repo['full_name'] == @github_repo } + end + + def repo_id + @repo_id ||= STDIN.gets.chomp.to_i + end + + def repos + Github::Repositories.new(@options).fetch + end +end + +namespace :import do + desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)' + task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args| + abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/') + + GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras) + end +end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 6ded519aff2..761f275d42a 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,7 +1,9 @@ require Rails.root.join('db/migrate/limits_to_mysql') +require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do puts "Adding limits to schema.rb for mysql" LimitsToMysql.new.up + MarkdownCacheLimitsToMysql.new.up end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 8938bc515f5..4108cee08b4 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') +require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do @@ -11,4 +12,5 @@ task setup_postgresql: :environment do AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up IndexRoutesPathForLike.new.up + IndexRedirectRoutesPathForLike.new.up end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 602c60be828..2eddcb3c777 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -60,7 +60,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %w(rake gitlab:setup), - %w(rspec spec), + %w(rspec spec) ] run_commands(cmds) end |