diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-05-23 02:10:29 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-05-23 02:10:29 +0800 |
| commit | 1a4130d3a6cfb4956f8bb1186cc499ea549d8e18 (patch) | |
| tree | 076adcb3e6f3800a1a7bbc6809839d5cb3b3f372 /lib | |
| parent | 3c8a6fba67998eb17240b15db85f8d1c8aff338e (diff) | |
| parent | 18a6d9c5326bc2b90a1f0cc8664d638a39885924 (diff) | |
| download | gitlab-ce-27377-preload-pipeline-entity.tar.gz | |
Merge remote-tracking branch 'upstream/master' into 27377-preload-pipeline-entity27377-preload-pipeline-entity
* upstream/master: (2534 commits)
Update VERSION to 9.3.0-pre
Update CHANGELOG.md for 9.2.0
removes unnecessary redundacy in usage ping doc
Respect the typo as rubocop said
Add a test to ensure this works on MySQL
Change pipelines schedules help page path
change domain to hostname in usage ping doc
Fixes broken MySQL migration for retried
Show password field mask while editing service settings
Add notes for supported schedulers and cloud providers
Move environment monitoring to environments doc
Add docs for change of Cache/Artifact restore order"
Avoid resource intensive login checks if password is not provided
Change translation for 'coding' by 'desarrollo' for Spanish
Add to docs: issues multiple assignees
rename "Add emoji" and "Award emoji" to "Add reaction" where appropriate
Add project and group notification settings info
32570 Fix border-bottom for project activity tab
Add users endpoint to frontend API class
Rename users on mysql
...
Diffstat (limited to 'lib')
268 files changed, 5666 insertions, 1389 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 1bf20f76ad6..52cd7cbe3db 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,6 +6,7 @@ module API version 'v3', using: :path do helpers ::API::V3::Helpers + helpers ::API::Helpers::CommonHelpers mount ::API::V3::AwardEmoji mount ::API::V3::Boards @@ -44,6 +45,9 @@ module API end before { allow_access_with_scope :api } + before { Gitlab::I18n.set_locale(current_user) } + + after { Gitlab::I18n.reset_locale } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -77,6 +81,7 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::SentryHelper helpers ::API::Helpers + helpers ::API::Helpers::CommonHelpers # Keep in alphabetical order mount ::API::AccessRequests diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 409cb5b924f..9fcf04efa38 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -121,7 +121,7 @@ module API end def oauth2_bearer_token_error_handler - Proc.new do |e| + proc do |e| response = case e when MissingTokenError diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 66b37fd2bcc..621b9dcecd9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,7 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -140,7 +140,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b888ede6fe8..8a54f7f3f05 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -47,6 +47,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/deploy_keys" do params[:key].strip! diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5954aea8041..01cc8e8e1ca 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -5,7 +5,10 @@ module API end class UserBasic < UserSafe - expose :id, :state, :avatar_url + expose :id, :state + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -14,10 +17,15 @@ module API class User < UserBasic expose :created_at - expose :is_admin?, as: :is_admin expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +33,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity @@ -34,8 +43,9 @@ module API expose :external end - class UserWithPrivateToken < UserPublic + class UserWithPrivateDetails < UserPublic expose :private_token + expose :admin?, as: :is_admin end class Email < Grape::Entity @@ -43,14 +53,14 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events expose :enable_ssl_verification end class ProjectHook < Hook expose :project_id, :issues_events, :merge_requests_events expose :note_events, :pipeline_events, :wiki_page_events - expose :build_events, as: :job_events + expose :job_events end class BasicProjectDetails < Grape::Entity @@ -90,7 +100,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -134,7 +146,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -184,19 +198,15 @@ module API end expose :protected do |repo_branch, options| - options[:project].protected_branch?(repo_branch.name) + ProtectedBranch.protected?(options[:project], repo_branch.name) end expose :developers_can_push do |repo_branch, options| - project = options[:project] - access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } + options[:project].protected_branches.developers_can?(:push, repo_branch.name) end expose :developers_can_merge do |repo_branch, options| - project = options[:project] - access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten - access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } + options[:project].protected_branches.developers_can?(:merge, repo_branch.name) end end @@ -204,7 +214,7 @@ module API expose :id, :name, :type, :path expose :mode do |obj, options| - filemode = obj.mode.to_s(8) + filemode = obj.mode filemode = "0" + filemode if filemode.length < 6 filemode end @@ -253,7 +263,11 @@ module API class IssueBasic < ProjectEntity expose :label_names, as: :labels expose :milestone, using: Entities::Milestone - expose :assignee, :author, using: Entities::UserBasic + expose :assignees, :author, using: Entities::UserBasic + + expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + issue.assignees.first + end expose :user_notes_count expose :upvotes, :downvotes @@ -456,7 +470,7 @@ module API expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events expose :tag_push_events, :note_events, :pipeline_events - expose :build_events, as: :job_events + expose :job_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -581,6 +595,7 @@ module API expose :plantuml_enabled expose :plantuml_url expose :terminal_max_session_time + expose :polling_interval_multiplier end class Release < Grape::Entity @@ -614,9 +629,9 @@ module API expose :locked expose :version, :revision, :platform, :architecture expose :contacted_at - expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } + expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } expose :projects, with: Entities::BasicProjectDetails do |runner, options| - if options[:current_user].is_admin? + if options[:current_user].admin? runner.projects else options[:current_user].authorized_projects.where(id: runner.projects) diff --git a/lib/api/files.rb b/lib/api/files.rb index 33fc970dc09..e6ea12c5ab7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,7 +5,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -130,7 +130,7 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] != :success render_api_error!(result[:message], 400) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 8f3799417e3..3da7d735da8 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -5,11 +5,16 @@ module API before { authenticate! } helpers do - params :optional_params do + params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' + end + + params :optional_params do + use :optional_params_ce end params :statistics_params do @@ -19,7 +24,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -47,7 +52,7 @@ module API elsif current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end @@ -56,7 +61,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.is_admin? + present_groups groups, statistics: params[:statistics] && current_user.admin? end desc 'Create a group. Available only for users who can create groups.' do @@ -142,7 +147,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 61527c1e20b..226a7ddd50e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,8 @@ module API end def find_project_snippet(id) - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params).find(id) + finder_params = { project: user_project } + SnippetsFinder.new(current_user, finder_params).execute.find(id) end def find_merge_request_with_access(iid, access_level = :read_merge_request) @@ -102,7 +102,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(current_user, :access_api) + unauthorized! unless current_user && can?(initial_current_user, :access_api) end def authenticate_non_get! @@ -118,7 +118,7 @@ module API def authenticated_as_admin! authenticate! - forbidden! unless current_user.is_admin? + forbidden! unless current_user.admin? end def authorize!(action, subject = :global) @@ -301,7 +301,7 @@ module API UploadedFile.new( file_path, params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', + params["#{field}.type"] || 'application/octet-stream' ) end @@ -358,7 +358,7 @@ module API return unless sudo_identifier return unless initial_current_user - unless initial_current_user.is_admin? + unless initial_current_user.admin? forbidden!('Must be admin to use sudo') end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb new file mode 100644 index 00000000000..322624c6092 --- /dev/null +++ b/lib/api/helpers/common_helpers.rb @@ -0,0 +1,13 @@ +module API + module Helpers + module CommonHelpers + def convert_parameters_from_legacy_format(params) + params.tap do |params| + if params[:assignee_id].present? + params[:assignee_ids] = [params.delete(:assignee_id)] + end + end + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 2135a787b11..264df7271a3 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,48 +1,14 @@ module API module Helpers module InternalHelpers - # Project paths may be any of the following: - # * /repository/storage/path/namespace/project - # * /namespace/project - # * namespace/project - # - # In addition, they may have a '.git' extension and multiple namespaces - # - # Transform all these cases to 'namespace/project' - def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values) - project_path = project_path.sub(/\.git\z/, '') - - storages.each do |storage| - storage_path = File.expand_path(storage['path']) - - if project_path.start_with?(storage_path) - project_path = project_path.sub(storage_path, '') - break - end - end - - project_path.sub(/\A\//, '') - end - - def project_path - @project_path ||= clean_project_path(params[:project]) - end - def wiki? - @wiki ||= project_path.end_with?('.wiki') && - !Project.find_by_full_path(project_path) + set_project unless defined?(@wiki) + @wiki end def project - @project ||= begin - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - Project.find_by_full_path(project_path) - end + set_project unless defined?(@project) + @project end def ssh_authentication_abilities @@ -53,12 +19,28 @@ module API ] end - def parse_allowed_environment_variables - return if params[:env].blank? + def parse_env + return {} if params[:env].blank? JSON.parse(params[:env]) - rescue JSON::ParserError + {} + end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end + + private + + def set_project + if params[:gl_repository] + @project, @wiki = Gitlab::GlRepository.parse(params[:gl_repository]) + else + @project, @wiki = Gitlab::RepoPath.parse(params[:project]) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 74848a6e144..1369b021ea4 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -50,10 +50,14 @@ module API forbidden!('Job has been erased!') if job.erased? end - def authenticate_job!(job) + def authenticate_job! + job = Ci::Build.find_by_id(params[:id]) + validate_job!(job) do forbidden! unless job_token_valid?(job) end + + job end def job_token_valid?(job) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 523f38d129e..96aaaf868ea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -11,14 +11,16 @@ module API # Params: # key_id - ssh key id for Git over SSH # user_id - user id for Git over HTTP + # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project path with namespace # action - git action (git-upload-pack or git-receive-pack) - # ref - branch name - # forced_push - forced_push - # protocol - Git access protocol being used, e.g. HTTP or SSH + # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList post "/allowed" do status 200 + # Stores some Git-specific env thread-safely + Gitlab::Git::Env.set(parse_env) + actor = if params[:key_id] Key.find_by(id: params[:key_id]) @@ -30,22 +32,20 @@ module API actor.update_last_used_at if actor.is_a?(Key) - access = - if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) - else - Gitlab::GitAccess.new(actor, - project, - protocol, - authentication_abilities: ssh_authentication_abilities, - env: parse_allowed_environment_variables) - end - - access_status = access.check(params[:action], params[:changes]) + access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_status = access_checker + .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + .check(params[:action], params[:changes]) response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = @@ -90,7 +90,7 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION, + gitlab_rev: Gitlab::REVISION } end @@ -139,9 +139,10 @@ module API return unless Gitlab::GitalyClient.enabled? begin - Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive + repository = wiki? ? project.wiki.repository : project.repository + Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive rescue GRPC::Unavailable => e - render_api_error(e, 500) + render_api_error!(e, 500) end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4dce5dd130a..78db960ae28 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -26,17 +26,23 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' + optional :search, type: String, desc: 'Search issues for text present in the title or description' use :pagination end - params :issue_params do + params :issue_params_ce do optional :description, type: String, desc: 'The description of an issue' - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' + optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end + + params :issue_params do + use :issue_params_ce + end end resource :issues do @@ -130,6 +136,8 @@ module API issue_params = declared_params(include_missing: false) + issue_params = convert_parameters_from_legacy_format(issue_params) + issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute @@ -154,7 +162,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do @@ -168,6 +176,8 @@ module API update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) + issue = ::Issues::UpdateService.new(user_project, current_user, update_params).execute(issue) @@ -214,6 +224,21 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + desc 'List merge requests closing issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/closed_by' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id) + merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids) + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end end end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ffab0aafe59..0223957fde1 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -118,7 +118,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end @@ -132,6 +132,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) build.cancel @@ -148,6 +149,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -165,6 +167,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -181,6 +184,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -201,6 +205,7 @@ module API build = get_build!(params[:job_id]) + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -211,12 +216,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c8033664133..710deba5ae3 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -20,6 +20,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -33,13 +35,28 @@ module API end end - params :optional_params do + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' end + + params :optional_params do + use :optional_params_ce + end end desc 'List merge requests' do @@ -53,23 +70,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end - - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end @@ -145,14 +154,24 @@ module API success Entities::MergeRequest end params do + # CE + at_least_one_of_ce = [ + :assignee_id, + :description, + :labels, + :milestone_id, + :remove_source_branch, + :state_event, + :target_branch, + :title + ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], desc: 'Status of the merge request' + use :optional_params - at_least_one_of :title, :target_branch, :description, :assignee_id, - :milestone_id, :labels, :state_event, - :remove_source_branch + at_least_one_of(*at_least_one_of_ce) end put ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) @@ -173,6 +192,7 @@ module API success Entities::MergeRequest end params do + # CE optional :merge_commit_message, type: String, desc: 'Custom merge commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' @@ -182,14 +202,15 @@ module API end put ':id/merge_requests/:merge_request_iid/merge' do merge_request = find_project_merge_request(params[:merge_request_iid]) + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! unless merge_request.mergeable_state? + not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) if params[:sha] && merge_request.diff_head_sha != params[:sha] render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) @@ -200,7 +221,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index e7ab82f08db..a3ea619a2fb 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -139,7 +139,7 @@ module API finder_params = { project_id: user_project.id, - milestone_id: milestone.id, + milestone_title: milestone.title, sort: 'position_asc' } diff --git a/lib/api/notes.rb b/lib/api/notes.rb index de39e579ac3..e281e3230fd 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -78,7 +78,7 @@ module API } if can?(current_user, noteable_read_ability_name(noteable), noteable) - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 754c3d85a04..9117704aa46 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -14,13 +14,23 @@ module API end params do use :pagination - optional :scope, type: String, values: %w(running branches tags), - desc: 'Either running, branches, or tags' + optional :scope, type: String, values: %w[running pending finished branches tags], + desc: 'The scope of pipelines' + optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, + desc: 'The status of pipelines' + optional :ref, type: String, desc: 'The ref of pipelines' + optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' + optional :name, type: String, desc: 'The name of the user who triggered pipelines' + optional :username, type: String, desc: 'The username of the user who triggered pipelines' + optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', + desc: 'Order pipelines' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Sort pipelines' end get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 53791166c33..7a345289617 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -13,7 +13,7 @@ module API optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" - optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" @@ -53,7 +53,9 @@ module API use :project_hook_properties end post ":id/hooks" do - hook = user_project.hooks.new(declared_params(include_missing: false)) + hook_params = declared_params(include_missing: false) + + hook = user_project.hooks.new(hook_params) if hook.save present hook, with: Entities::ProjectHook @@ -74,7 +76,9 @@ module API put ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - if hook.update_attributes(declared_params(include_missing: false)) + update_params = declared_params(include_missing: false) + + if hook.update_attributes(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index cfee38a9baf..98bc9c28527 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -17,8 +17,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0fbe1669d45..ed5004e8d1a 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -6,12 +6,12 @@ module API before { authenticate_non_get! } helpers do - params :optional_params do + params :optional_params_ce do optional :description, type: String, desc: 'The description of the project' optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -22,6 +22,14 @@ module API optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' end + + params :optional_params do + use :optional_params_ce + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end end resource :projects do @@ -52,10 +60,6 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' @@ -65,7 +69,7 @@ module API options = options.reverse_merge( with: Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -81,10 +85,11 @@ module API end params do use :collection_params + use :statistics_params end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] end desc 'Create new project' do @@ -99,6 +104,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -146,10 +152,13 @@ module API desc 'Get a single project' do success Entities::ProjectWithAccess end + params do + use :statistics_params + end get ":id" do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end desc 'Get events for a single project' do @@ -198,17 +207,33 @@ module API success Entities::Project end params do + # CE + at_least_one_of_ce = + [ + :jobs_enabled, + :container_registry_enabled, + :default_branch, + :description, + :issues_enabled, + :lfs_enabled, + :merge_requests_enabled, + :name, + :only_allow_merge_if_all_discussions_are_resolved, + :only_allow_merge_if_pipeline_succeeds, + :path, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :visibility, + :wiki_enabled + ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' optional :path, type: String, desc: 'The path of the repository' + use :optional_params - at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, - :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :visibility, :public_builds, - :request_access_enabled, :only_allow_merge_if_pipeline_succeeds, - :only_allow_merge_if_all_discussions_are_resolved, :path, - :default_branch + at_least_one_of(*at_least_one_of_ce) end put ':id' do authorize_admin_project @@ -216,6 +241,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success @@ -358,7 +385,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 4c9db2c8716..6fbb02cb3aa 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -113,10 +113,9 @@ module API optional :state, type: String, desc: %q(Job's status: success, failed) end put '/:id' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! - job.update_attributes(trace: params[:trace]) if params[:trace] + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.path_with_namespace) @@ -140,23 +139,20 @@ module API optional :token, type: String, desc: %q(Job's authentication token) end patch '/:id/trace' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = job.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = job.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - job.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Job-Status', job.status - header 'Range', "0-#{job.trace_length}" + header 'Range', "0-#{stream_size}" end desc 'Authorize artifacts uploading for job' do @@ -175,8 +171,7 @@ module API require_gitlab_workhorse! Gitlab::Workhorse.verify_api_request!(headers) - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! forbidden!('Job is not running') unless job.running? if params[:filesize] @@ -212,8 +207,7 @@ module API not_allowed! unless Gitlab.config.artifacts.enabled require_gitlab_workhorse! - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! forbidden!('Job is not running!') unless job.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path @@ -245,8 +239,7 @@ module API optional :token, type: String, desc: %q(Job's authentication token) end get '/:id/artifacts' do - job = Ci::Build.find_by_id(params[:id]) - authenticate_job!(job) + job = authenticate_job! artifacts_file = job.artifacts_file unless artifacts_file.file_storage? diff --git a/lib/api/runners.rb b/lib/api/runners.rb index a77c876a749..db6c7c59092 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -161,18 +161,18 @@ module API end def authenticate_show_runner!(runner) - return if runner.is_shared || current_user.is_admin? + return if runner.is_shared || current_user.admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_update_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_delete_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) @@ -181,7 +181,7 @@ module API def authenticate_enable_runner!(runner) forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner is locked") if runner.locked? - return if current_user.is_admin? + return if current_user.admin? forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 4e0c9cb1f63..cb07df9e249 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -356,7 +356,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -488,6 +488,14 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + } + ], 'mattermost' => [ { required: true, @@ -550,7 +558,8 @@ module API RedmineService, SlackService, MattermostService, - TeamcityService, + MicrosoftTeamsService, + TeamcityService ] if Rails.env.development? @@ -562,8 +571,14 @@ module API desc: 'URL to the mock service' } ] + services['mock-deployment'] = [] + services['mock-monitoring'] = [] - service_classes << MockCiService + service_classes += [ + MockCiService, + MockDeploymentService, + MockMonitoringService + ] end trigger_services = { @@ -627,7 +642,7 @@ module API service_params = declared_params(include_missing: false).merge(active: true) if service.update_attributes(service_params) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? else render_api_error!('400 Bad Request', 400) end @@ -658,7 +673,7 @@ module API end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? end end diff --git a/lib/api/session.rb b/lib/api/session.rb index 002ffd1d154..016415c3023 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserWithPrivateToken + success Entities::UserWithPrivateDetails end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateToken + present user, with: Entities::UserWithPrivateDetails end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d4d3229f0d1..82f513c984e 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -20,6 +20,56 @@ module API success Entities::ApplicationSetting end params do + # CE + at_least_one_of_ce = [ + :admin_notification_email, + :after_sign_out_path, + :after_sign_up_text, + :akismet_enabled, + :container_registry_token_expire_delay, + :default_artifacts_expire_in, + :default_branch_protection, + :default_group_visibility, + :default_project_visibility, + :default_projects_limit, + :default_snippet_visibility, + :disabled_oauth_sign_in_sources, + :domain_blacklist_enabled, + :domain_whitelist, + :email_author_in_body, + :enabled_git_access_protocol, + :gravatar_enabled, + :help_page_text, + :home_page_url, + :housekeeping_enabled, + :html_emails_enabled, + :import_sources, + :koding_enabled, + :max_artifacts_size, + :max_attachment_size, + :max_pages_size, + :metrics_enabled, + :plantuml_enabled, + :polling_interval_multiplier, + :recaptcha_enabled, + :repository_checks_enabled, + :repository_storage, + :require_two_factor_authentication, + :restricted_visibility_levels, + :send_user_confirmation_email, + :sentry_enabled, + :clientside_sentry_enabled, + :session_expire_delay, + :shared_runners_enabled, + :sidekiq_throttling_enabled, + :sign_in_text, + :signin_enabled, + :signup_enabled, + :terminal_max_session_time, + :user_default_external, + :user_oauth_applications, + :version_check_enabled + ] optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' @@ -89,6 +139,10 @@ module API given sentry_enabled: ->(val) { val } do requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' end + optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' + given clientside_sentry_enabled: ->(val) { val } do + requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' + end optional :repository_storage, type: String, desc: 'Storage paths for new projects' optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :koding_enabled, type: Boolean, desc: 'Enable Koding' @@ -110,22 +164,9 @@ module API requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' - at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, - :default_group_visibility, :restricted_visibility_levels, :import_sources, - :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, - :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, - :user_oauth_applications, :user_default_external, :signup_enabled, - :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, - :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, - :shared_runners_enabled, :max_artifacts_size, - :default_artifacts_expire_in, :max_pages_size, - :container_registry_token_expire_delay, - :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, - :akismet_enabled, :admin_notification_email, :sentry_enabled, - :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, - :version_check_enabled, :email_author_in_body, :html_emails_enabled, - :housekeeping_enabled, :terminal_max_session_time + optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + + at_least_one_of(*at_least_one_of_ce) end put "application/settings" do attrs = declared_params(include_missing: false) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b93fdc62808..53f5953a8fb 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index dbe54d3cd31..91567909998 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -5,7 +5,7 @@ module API subscribable_types = { 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/users.rb b/lib/api/users.rb index a4201fe6fed..3d83720b7b9 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -37,11 +37,16 @@ module API success Entities::UserBasic end params do + # CE optional :username, type: String, desc: 'Get a single user with a specific username' + optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + all_or_none_of :extern_uid, :provider + use :pagination end get do @@ -49,17 +54,11 @@ module API render_api_error!("Not authorized.", 403) end - if params[:username].present? - users = User.where(username: params[:username]) - else - users = User.all - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - users = users.external if params[:external] && current_user.is_admin? - end + authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) - entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic + users = UsersFinder.new(current_user, params).execute + + entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic present paginate(users), with: entity end @@ -73,7 +72,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - if current_user && current_user.is_admin? + if current_user && current_user.admin? present user, with: Entities::UserPublic elsif can?(current_user, :read_user, user) present user, with: Entities::User @@ -293,7 +292,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - ::Users::DestroyService.new(current_user).execute(user) + DeleteUserWorker.perform_async(current_user.id, user.id) end desc 'Block a user. Available only for admins.' @@ -341,7 +340,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent @@ -425,7 +424,7 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do @@ -532,6 +531,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 6f97102c6ef..21935922414 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -120,7 +120,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end @@ -134,6 +134,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) build.cancel @@ -150,6 +151,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -167,6 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -183,6 +186,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -202,7 +206,7 @@ module API authorize_read_builds! build = get_build!(params[:build_id]) - + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -213,12 +217,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 3414a2883e5..674de592f0a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -53,7 +53,7 @@ module API attrs = declared_params.dup branch = attrs.delete(:branch_name) - attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + attrs.merge!(start_branch: branch, branch_name: branch) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -131,7 +131,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 832b4bdeb4f..332f233bf5e 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -69,7 +69,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -129,7 +131,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -234,7 +238,8 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :build_events, :pipeline_events + expose :tag_push_events, :note_events, :pipeline_events + expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -246,7 +251,15 @@ module API class ProjectHook < ::API::Entities::Hook expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :build_events, :pipeline_events, :wiki_page_events + expose :note_events, :pipeline_events, :wiki_page_events + expose :job_events, as: :build_events + end + + class Issue < ::API::Entities::Issue + unexpose :assignees + expose :assignee do |issue, options| + ::API::Entities::UserBasic.represent(issue.assignees.first, options) + end end end end diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb index 13542b0c71c..c76acc86504 100644 --- a/lib/api/v3/files.rb +++ b/lib/api/v3/files.rb @@ -6,7 +6,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -123,7 +123,7 @@ module API file_params = declared_params(include_missing: false) file_params[:branch] = file_params.delete(:branch_name) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index c5b37622d79..6187445fc8d 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -20,7 +20,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -45,7 +45,7 @@ module API groups = if current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end @@ -54,7 +54,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.is_admin? + present_groups groups, statistics: params[:statistics] && current_user.admin? end desc 'Get list of owned groups for authenticated user' do @@ -151,7 +151,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index 715083fc4f8..cb371fdbab8 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -8,6 +8,7 @@ module API helpers do def find_issues(args = {}) args = params.merge(args) + args = convert_parameters_from_legacy_format(args) args.delete(:id) args[:milestone_title] = args.delete(:milestone) @@ -51,7 +52,7 @@ module API resource :issues do desc "Get currently authenticated user's issues" do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -61,7 +62,7 @@ module API get do issues = find_issues(scope: 'authored') - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -70,7 +71,7 @@ module API end resource :groups, requirements: { id: %r{[^/]+} } do desc 'Get a list of group issues' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -82,7 +83,7 @@ module API issues = find_issues(group_id: group.id, match_all_labels: true) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -94,7 +95,7 @@ module API desc 'Get a list of project issues' do detail 'iid filter is deprecated have been removed on V4' - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -107,22 +108,22 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Get a single project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' end get ":id/issues/:issue_id" do issue = find_project_issue(params[:issue_id]) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Create a new project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :title, type: String, desc: 'The title of an issue' @@ -140,6 +141,7 @@ module API issue_params = declared_params(include_missing: false) issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions)) + issue_params = convert_parameters_from_legacy_format(issue_params) issue = ::Issues::CreateService.new(user_project, current_user, @@ -147,14 +149,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Update an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -176,6 +178,7 @@ module API end update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) issue = ::Issues::UpdateService.new(user_project, current_user, @@ -184,14 +187,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Move an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -206,7 +209,7 @@ module API begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 3077240e650..b6b7254ae29 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -23,6 +23,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -32,7 +34,7 @@ module API if project.has_external_issue_tracker? ::API::Entities::ExternalIssue else - ::API::Entities::Issue + ::API::V3::Entities::Issue end end diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index be90cec4afc..4c7061d4939 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -39,7 +39,7 @@ module API end desc 'Get all issues for a single project milestone' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -56,7 +56,7 @@ module API } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end end end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb index 4f8e0eff4ff..009ec5c6bbd 100644 --- a/lib/api/v3/notes.rb +++ b/lib/api/v3/notes.rb @@ -79,7 +79,7 @@ module API noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) if can?(current_user, noteable_read_ability_name(noteable), noteable) - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index 82827249244..c48cbd2b765 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -21,7 +21,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute present paginate(pipelines), with: ::API::Entities::Pipeline end end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index fc065a22d74..c41fee32610 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -18,8 +18,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index b753dbab381..164612cb8dd 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -88,7 +88,7 @@ module API options = options.reverse_merge( with: ::API::V3::Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -107,7 +107,7 @@ module API end get '/visible' do entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity end desc 'Get a projects list for authenticated user' do @@ -452,7 +452,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index 1934d6e578c..faa265f3314 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -50,7 +50,7 @@ module API helpers do def authenticate_delete_runner!(runner) - return if current_user.is_admin? + return if current_user.admin? forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 3bacaeee032..118c6df6549 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -377,7 +377,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -501,6 +501,12 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + ], 'mattermost' => [ { required: true, @@ -596,7 +602,7 @@ module API end get ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + present service, with: Entities::ProjectService, include_passwords: current_user.admin? end end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 07dac7e9904..0762fc02d70 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb index 068750ec077..690768db82f 100644 --- a/lib/api/v3/subscriptions.rb +++ b/lib/api/v3/subscriptions.rb @@ -7,7 +7,7 @@ module API 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 5e18cecc431..f4cda3b2eba 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -138,7 +138,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4016ac76348..d97e5d98229 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -80,16 +80,32 @@ module Backup 'port' => '--port', 'socket' => '--socket', 'username' => '--user', - 'encoding' => '--default-character-set' + 'encoding' => '--default-character-set', + # SSL + 'sslkey' => '--ssl-key', + 'sslcert' => '--ssl-cert', + 'sslca' => '--ssl-ca', + 'sslcapath' => '--ssl-capath', + 'sslcipher' => '--ssl-cipher' } args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + args = { + 'username' => 'PGUSER', + 'host' => 'PGHOST', + 'port' => 'PGPORT', + 'password' => 'PGPASSWORD', + # SSL + 'sslmode' => 'PGSSLMODE', + 'sslkey' => 'PGSSLKEY', + 'sslcert' => 'PGSSLCERT', + 'sslrootcert' => 'PGSSLROOTCERT', + 'sslcrl' => 'PGSSLCRL', + 'sslcompression' => 'PGSSLCOMPRESSION' + } + args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] } end def report_success(success) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..330cd963626 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -15,11 +15,10 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(Gitlab.config.backup.path) do - File.open("#{Gitlab.config.backup.path}/backup_information.yml", - "w+") do |file| + Dir.chdir(backup_path) do + File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/, '') end @@ -64,9 +63,9 @@ module Backup $progress.print "Deleting tmp directories ... " backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + next unless File.exist?(File.join(backup_path, dir)) - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(backup_path, dir)) $progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) @@ -83,8 +82,8 @@ module Backup if keep_time > 0 removed = 0 - Dir.chdir(Gitlab.config.backup.path) do - Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + Dir.chdir(backup_path) do + backup_file_list.each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -107,18 +106,14 @@ module Backup end def unpack - Dir.chdir(Gitlab.config.backup.path) + Dir.chdir(backup_path) # check for existing backups in the backup dir - file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") - - if file_list.count == 0 - $progress.puts "No backups found in #{Gitlab.config.backup.path}" + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - end - - if file_list.count > 1 && ENV["BACKUP"].nil? + elsif backup_file_list.many? && ENV["BACKUP"].nil? $progress.puts 'Found more than one backup, please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 @@ -127,7 +122,7 @@ module Backup tar_file = if ENV['BACKUP'].present? "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" else - file_list.first + backup_file_list.first end unless File.exist?(tar_file) @@ -169,6 +164,14 @@ module Backup private + def backup_path + Gitlab.config.backup.path + end + + def backup_file_list + @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") + end + def connect_to_remote_directory(connection_settings) connection = ::Fog::Storage.new(connection_settings) diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d6138816e70..6255a611dbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -53,7 +53,10 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): + (?=[^[:alnum:]:]|$)/x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index d67d466bce8..d6327ef31cb 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -2,16 +2,17 @@ module Banzai module Filter # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter + SCHEMES = ['http', 'https', nil].freeze + def call links.each do |node| - href = href_to_lowercase_scheme(node["href"].to_s) + uri = uri(node['href'].to_s) + next unless uri - unless node["href"].to_s == href - node.set_attribute('href', href) - end + node.set_attribute('href', uri.to_s) - if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) - node.set_attribute('rel', 'nofollow noreferrer') + if SCHEMES.include?(uri.scheme) && external_url?(uri) + node.set_attribute('rel', 'nofollow noreferrer noopener') node.set_attribute('target', '_blank') end end @@ -21,27 +22,26 @@ module Banzai private + def uri(href) + URI.parse(href) + rescue URI::Error + nil + end + def links query = 'descendant-or-self::a[@href and not(@href = "")]' doc.xpath(query) end - def href_to_lowercase_scheme(href) - scheme_match = href.match(/\A(\w+):\/\//) - - if scheme_match - scheme_match.to_s.downcase + scheme_match.post_match - else - href - end - end + def external_url?(uri) + # Relative URLs miss a hostname + return false unless uri.hostname - def external_url?(url) - !url.start_with?(internal_url) + uri.hostname != internal_url.hostname end def internal_url - @internal_url ||= Gitlab.config.gitlab.url + @internal_url ||= URI.parse(Gitlab.config.gitlab.url) end end end diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb new file mode 100644 index 00000000000..327ea9449a1 --- /dev/null +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -0,0 +1,37 @@ +module Banzai + module Filter + # HTML filter that appends state information to issuable links. + # Runs as a post-process filter as issuable state might change whilst + # Markdown is in the cache. + # + # This filter supports cross-project references. + class IssuableStateFilter < HTML::Pipeline::Filter + VISIBLE_STATES = %w(closed merged).freeze + + def call + return doc unless context[:issuable_state_filter_enabled] + + extractor = Banzai::IssuableExtractor.new(project, current_user) + issuables = extractor.extract([doc]) + + issuables.each do |node, issuable| + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" + end + end + + doc + end + + private + + def current_user + context[:current_user] + end + + def project + context[:project] + end + end + end +end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ff580ec68f8..ee73fa91589 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -14,7 +14,7 @@ module Banzai def self.renderer @renderer ||= begin - renderer = Redcarpet::Render::HTML.new + renderer = Banzai::Renderer::HTML.new Redcarpet::Markdown.new(renderer, redcarpet_options) end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c59a80dd1c7..9f9882b3b40 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,7 +7,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Redactor.new(project, current_user).redact([doc]) + Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction] doc end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d5f9e252f62..522217deae4 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -24,10 +24,6 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class v-pre) - whitelist[:attributes]['span'] = %w(class) - # Allow table alignment whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -52,9 +48,6 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) - # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(self.class.clean_spans) - whitelist end @@ -84,21 +77,6 @@ module Banzai end end end - - def clean_spans - lambda do |env| - node = env[:node] - - return unless node.name == 'span' - return unless node.has_attribute?('class') - - unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } - node.remove_attribute('class') - end - - { node_whitelist: [node] } - end - end end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9f09ca90697..7da565043d1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai end def highlight_node(node) - language = node.attr('class') + language = node.attr('lang') code = node.text css_classes = "code highlight" lexer = lexer_for(language) diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb new file mode 100644 index 00000000000..cbabf9156de --- /dev/null +++ b/lib/banzai/issuable_extractor.rb @@ -0,0 +1,40 @@ +module Banzai + # Extract references to issuables from multiple documents + + # This populates RequestStore cache used in Banzai::ReferenceParser::IssueParser + # and Banzai::ReferenceParser::MergeRequestParser + # Populating the cache should happen before processing documents one-by-one + # so we can avoid N+1 queries problem + + class IssuableExtractor + QUERY = %q( + descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] + [@data-reference-type="issue" or @data-reference-type="merge_request"] + ).freeze + + attr_reader :project, :user + + def initialize(project, user) + @project = project + @user = user + end + + # Returns Hash in the form { node => issuable_instance } + def extract(documents) + nodes = documents.flat_map do |document| + document.xpath(QUERY) + end + + issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) + merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) + + issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( + merge_request_parser.merge_requests_for_nodes(nodes) + ) + + # The project for the issue/MR might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes.select { |node, issuable| issuable.project } + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9f8eb0931b8..002a3341ccd 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -31,7 +31,8 @@ module Banzai # # Returns the same input objects. def render(objects, attribute) - documents = render_objects(objects, attribute) + documents = render_documents(objects, attribute) + documents = post_process_documents(documents, objects, attribute) redacted = redact_documents(documents) objects.each_with_index do |object, index| @@ -41,9 +42,24 @@ module Banzai end end - # Renders the attribute of every given object. - def render_objects(objects, attribute) - render_attributes(objects, attribute) + private + + def render_documents(objects, attribute) + pipeline = HTML::Pipeline.new([]) + + objects.map do |object| + pipeline.to_document(Banzai.render_field(object, attribute)) + end + end + + def post_process_documents(documents, objects, attribute) + # Called here to populate cache, refer to IssuableExtractor docs + IssuableExtractor.new(project, user).extract(documents) + + documents.zip(objects).map do |document, object| + context = context_for(object, attribute) + Banzai::Pipeline[:post_process].to_document(document, context) + end end # Redacts the list of documents. @@ -57,25 +73,15 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.dup - context = context.merge(object.banzai_render_context(attribute)) - context - end - - # Renders the attributes of a set of objects. - # - # Returns an Array of `Nokogiri::HTML::Document`. - def render_attributes(objects, attribute) - objects.map do |object| - string = Banzai.render_field(object, attribute) - context = context_for(object, attribute) - - Banzai::Pipeline[:relative_link].to_document(string, context) - end + base_context.merge(object.banzai_render_context(attribute)) end def base_context - @base_context ||= @redaction_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge( + current_user: user, + project: project, + skip_redaction: true + ) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index fd4a6a107c2..bd4d1aa9ff8 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -9,9 +9,9 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ - Filter::SyntaxHighlightFilter, Filter::PlantumlFilter, Filter::SanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::UploadLinkFilter, diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb new file mode 100644 index 00000000000..c56d908009f --- /dev/null +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -0,0 +1,13 @@ +module Banzai + module Pipeline + class MarkupPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::PlantumlFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index ecff094b1e5..131ac3b0eec 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -4,6 +4,7 @@ module Banzai def self.filters FilterArray[ Filter::RelativeLinkFilter, + Filter::IssuableStateFilter, Filter::RedactorFilter ] end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 52fdb9a2140..c2503fa2adc 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -62,8 +62,7 @@ module Banzai nodes.select do |node| if node.has_attribute?(project_attr) - node_id = node.attr(project_attr).to_i - can_read_reference?(user, projects[node_id]) + can_read_reference?(user, projects[node]) else true end @@ -112,12 +111,12 @@ module Banzai per_project end - # Returns a Hash containing objects for an attribute grouped per their - # IDs. + # Returns a Hash containing objects for an attribute grouped per the + # nodes that reference them. # # The returned Hash uses the following format: # - # { id value => row } + # { node => row } # # nodes - An Array of HTML nodes to process. # @@ -132,9 +131,15 @@ module Banzai return {} if nodes.empty? ids = unique_attribute_values(nodes, attribute) - rows = collection_objects_for_ids(collection, ids) + collection_objects = collection_objects_for_ids(collection, ids) + objects_by_id = collection_objects.index_by(&:id) - rows.index_by(&:id) + nodes.each_with_object({}) do |node, hash| + if node.has_attribute?(attribute) + obj = objects_by_id[node.attr(attribute).to_i] + hash[node] = obj if obj + end + end end # Returns an Array containing all unique values of an attribute of the @@ -201,7 +206,7 @@ module Banzai # # The returned Hash uses the following format: # - # { project ID => project } + # { node => project } # def projects_for_nodes(nodes) @projects_for_nodes ||= diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 6c20dec5734..89ec715ddf6 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -13,14 +13,14 @@ module Banzai issues_readable_by_user(issues.values, user).to_set nodes.select do |node| - readable_issues.include?(issue_for_node(issues, node)) + readable_issues.include?(issues[node]) end end def referenced_by(nodes) issues = issues_for_nodes(nodes) - nodes.map { |node| issue_for_node(issues, node) }.uniq + nodes.map { |node| issues[node] }.compact.uniq end def issues_for_nodes(nodes) @@ -28,7 +28,7 @@ module Banzai nodes, Issue.all.includes( :author, - :assignee, + :assignees, { # These associations are primarily used for checking permissions. # Eager loading these ensures we don't end up running dozens of @@ -44,12 +44,6 @@ module Banzai self.class.data_attribute ) end - - private - - def issue_for_node(issues, node) - issues[node.attr(self.class.data_attribute).to_i] - end end end end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 40451947e6c..8b0662749fd 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -3,14 +3,42 @@ module Banzai class MergeRequestParser < BaseParser self.reference_type = :merge_request - def references_relation - MergeRequest.includes(:author, :assignee, :target_project) + def nodes_visible_to_user(user, nodes) + merge_requests = merge_requests_for_nodes(nodes) + + nodes.select do |node| + merge_request = merge_requests[node] + + merge_request && can?(user, :read_merge_request, merge_request.project) + end end - private + def referenced_by(nodes) + merge_requests = merge_requests_for_nodes(nodes) + + nodes.map { |node| merge_requests[node] }.compact.uniq + end - def can_read_reference?(user, ref_project) - can?(user, :read_merge_request, ref_project) + def merge_requests_for_nodes(nodes) + @merge_requests_for_nodes ||= grouped_objects_for_nodes( + nodes, + MergeRequest.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + target_project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members, + :project_feature + ] + }), + self.class.data_attribute + ) end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 7adaffa19c1..09b66cbd8fb 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -49,7 +49,7 @@ module Banzai # Check if project belongs to a group which # user can read. def can_read_group_reference?(node, user, groups) - node_group = groups[node.attr('data-group').to_i] + node_group = groups[node] node_group && can?(user, :read_group, node_group) end @@ -74,8 +74,8 @@ module Banzai if project && project_id && project.id == project_id.to_i true elsif project_id && user_id - project = projects[project_id.to_i] - user = users[user_id.to_i] + project = projects[node] + user = users[node] project && user ? project.team.member?(user) : false else diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb new file mode 100644 index 00000000000..252caa35947 --- /dev/null +++ b/lib/banzai/renderer/html.rb @@ -0,0 +1,13 @@ +module Banzai + module Renderer + class HTML < Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + end + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index 94adaacc9b5..800d5a075c6 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -1,6 +1,8 @@ module Bitbucket module Representation class Base + attr_reader :raw + def initialize(raw) @raw = raw end @@ -8,10 +10,6 @@ module Bitbucket def self.decorate(entries) entries.map { |entry| new(entry)} end - - private - - attr_reader :raw end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b3ccad7b28d..55402101e43 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -20,7 +20,7 @@ module Ci italic: 0x02, underline: 0x04, conceal: 0x08, - cross: 0x10, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) @@ -132,34 +132,54 @@ module Ci STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - def convert(raw, new_state) + def convert(stream, new_state) reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset open_new_tag - s = StringScanner.new(ansi) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size end - @offset += s.matched_size end close_open_tags() - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) end def handle_sequence(s) @@ -240,10 +260,10 @@ module Ci Base64.urlsafe_encode64(state.to_json) end - def restore_state(raw, new_state) + def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length + return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| send("#{param}=".to_sym, state[param]) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 746e76a1b1f..67b269b330c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -61,7 +61,7 @@ module Ci update_runner_info - build.update_attributes(trace: params[:trace]) if params[:trace] + build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: build.project.path_with_namespace) @@ -86,23 +86,20 @@ module Ci # Example Request: # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = build.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = build.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - build.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Build-Status', build.status - header 'Range', "0-#{build.trace_length}" + header 'Range', "0-#{stream_size}" end # Authorize artifacts uploading for build - Runners only @@ -117,8 +114,7 @@ module Ci require_gitlab_workhorse! Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! forbidden!('build is not running') unless build.running? if params[:filesize] @@ -154,8 +150,7 @@ module Ci post ":id/artifacts" do require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! forbidden!('Build is not running!') unless build.running? artifacts_upload_path = ArtifactUploader.artifacts_upload_path @@ -189,8 +184,7 @@ module Ci # Example Request: # GET /builds/:id/artifacts get ":id/artifacts" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -214,8 +208,7 @@ module Ci # Example Request: # DELETE /builds/:id/artifacts delete ":id/artifacts" do - build = Ci::Build.find_by_id(params[:id]) - authenticate_build!(build) + build = authenticate_build! status(200) build.erase_artifacts! diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 996990b464f..5109dc9670f 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,10 +13,14 @@ module Ci forbidden! unless current_runner end - def authenticate_build!(build) + def authenticate_build! + build = Ci::Build.find_by_id(params[:id]) + validate_build!(build) do forbidden! unless build_token_valid?(build) end + + build end def validate_build!(build) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 15a461a16dd..b06474cda7f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -70,7 +70,7 @@ module Ci cache: job[:cache], dependencies: job[:dependencies], after_script: job[:after_script], - environment: job[:environment], + environment: job[:environment] }.compact } end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index bae4db1ca4d..5f379756c11 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,16 +2,8 @@ class GroupUrlConstrainer def matches?(request) id = request.params[:id] - return false unless valid?(id) + return false unless DynamicPathValidator.valid?(id) - Group.find_by_full_path(id).present? - end - - private - - def valid?(id) - id.split('/').all? do |namespace| - NamespaceValidator.valid?(namespace) - end + Group.find_by_full_path(id, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index a10b4657d7d..6f542f63f98 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,10 +4,8 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = namespace_path + '/' + project_path - unless ProjectPathValidator.valid?(project_path) - return false - end + return false unless DynamicPathValidator.valid?(full_path) - Project.find_by_full_path(full_path).present? + Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 9ab5bcb12ff..28159dc0dec 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,5 +1,5 @@ class UserUrlConstrainer def matches?(request) - User.find_by_username(request.params[:username]).present? + User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present? end end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..d5f85f9fcad 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name, digest) + client.delete_blob(repository.path, digest) end def data - @data ||= client.blob(repository.name, digest, type) + @data ||= client.blob(repository.path, digest, type) end end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 7f5f6d9ddb6..c7263f302ab 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -75,10 +75,7 @@ module ContainerRegistry def redirect_response(location) return unless location - # We explicitly remove authorization token - faraday_blob.get(location) do |req| - req['Authorization'] = '' - end + faraday_redirect.get(location) end def faraday @@ -93,5 +90,14 @@ module ContainerRegistry initialize_connection(conn, @options) end end + + # Create a new request to make sure the Authorization header is not inserted + # via the Faraday middleware + def faraday_redirect + @faraday_redirect ||= Faraday.new(@base_uri) do |conn| + conn.request :json + conn.adapter :net_http + end + end end end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb new file mode 100644 index 00000000000..61849a40383 --- /dev/null +++ b/lib/container_registry/path.rb @@ -0,0 +1,76 @@ +module ContainerRegistry + ## + # Class responsible for extracting project and repository name from + # image repository path provided by a containers registry API response. + # + # Example: + # + # some/group/my_project/my/image -> + # project: some/group/my_project + # repository: my/image + # + class Path + InvalidRegistryPathError = Class.new(StandardError) + + LEVELS_SUPPORTED = 3 + + def initialize(path) + @path = path.to_s.downcase + end + + def valid? + @path =~ Gitlab::Regex.container_repository_name_regex && + components.size > 1 && + components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def components + @components ||= @path.split('/') + end + + def nodes + raise InvalidRegistryPathError unless valid? + + @nodes ||= components.size.downto(2).map do |length| + components.take(length).join('/') + end + end + + def has_project? + repository_project.present? + end + + def has_repository? + return false unless has_project? + + repository_project.container_repositories + .where(name: repository_name).any? + end + + def root_repository? + @path == project_path + end + + def repository_project + @project ||= Project + .where_full_path_in(nodes.first(LEVELS_SUPPORTED)) + .first + end + + def repository_name + return unless has_project? + + @path.remove(%r(^#{Regexp.escape(project_path)}/?)) + end + + def project_path + return unless has_project? + + repository_project.full_path.downcase + end + + def to_s + @path + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 0e634f6b6ef..63bce655f57 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -8,10 +8,6 @@ module ContainerRegistry @client = ContainerRegistry::Client.new(uri, options) end - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - private def default_path diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..728deea224f 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,15 +22,17 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.path, name) end def path "#{repository.path}:#{name}" end + def location + "#{repository.location}:#{name}" + end + def [](key) return unless manifest @@ -38,9 +40,7 @@ module ContainerRegistry end def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.name, name) + @digest ||= client.repository_tag_digest(repository.path, name) end def config_blob @@ -82,7 +82,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.path, digest) end end end diff --git a/lib/github/client.rb b/lib/github/client.rb new file mode 100644 index 00000000000..e65d908d232 --- /dev/null +++ b/lib/github/client.rb @@ -0,0 +1,23 @@ +module Github + class Client + attr_reader :connection, :rate_limit + + def initialize(options) + @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, 60) + faraday.options.timeout = options.fetch(:timeout, 60) + faraday.authorization 'token', options.fetch(:token) + faraday.adapter :net_http + end + + @rate_limit = RateLimit.new(connection) + end + + def get(url, query = {}) + exceed, reset_in = rate_limit.get + sleep reset_in if exceed + + Github::Response.new(connection.get(url, query)) + end + end +end diff --git a/lib/github/collection.rb b/lib/github/collection.rb new file mode 100644 index 00000000000..014b2038c4b --- /dev/null +++ b/lib/github/collection.rb @@ -0,0 +1,29 @@ +module Github + class Collection + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch(url, query = {}) + return [] if url.blank? + + Enumerator.new do |yielder| + loop do + response = client.get(url, query) + response.body.each { |item| yielder << item } + + raise StopIteration unless response.rels.key?(:next) + url = response.rels[:next] + end + end.lazy + end + + private + + def client + @client ||= Github::Client.new(options) + end + end +end diff --git a/lib/github/error.rb b/lib/github/error.rb new file mode 100644 index 00000000000..66d7afaa787 --- /dev/null +++ b/lib/github/error.rb @@ -0,0 +1,3 @@ +module Github + RepositoryFetchError = Class.new(StandardError) +end diff --git a/lib/github/import.rb b/lib/github/import.rb new file mode 100644 index 00000000000..9c7eb965f93 --- /dev/null +++ b/lib/github/import.rb @@ -0,0 +1,412 @@ +require_relative 'error' + +module Github + class Import + include Gitlab::ShellAdapter + + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Note < ::Note + self.table_name = 'notes' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose + + def initialize(project, options) + @project = project + @repository = project.repository + @repo = project.import_source + @options = options + @verbose = options.fetch(:verbose, false) + @cached = Hash.new { |hash, key| hash[key] = Hash.new } + @errors = [] + end + + # rubocop: disable Rails/Output + def execute + puts 'Fetching repository...'.color(:aqua) if verbose + fetch_repository + puts 'Fetching labels...'.color(:aqua) if verbose + fetch_labels + puts 'Fetching milestones...'.color(:aqua) if verbose + fetch_milestones + puts 'Fetching pull requests...'.color(:aqua) if verbose + fetch_pull_requests + puts 'Fetching issues...'.color(:aqua) if verbose + fetch_issues + puts 'Cloning wiki repository...'.color(:aqua) if verbose + fetch_wiki_repository + puts 'Expiring repository cache...'.color(:aqua) if verbose + expire_repository_cache + + true + rescue Github::RepositoryFetchError + false + ensure + keep_track_of_errors + end + + private + + def fetch_repository + begin + project.create_repository unless project.repository.exists? + project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.set_remote_as_mirror('github') + project.repository.fetch_remote('github', forced: true) + rescue Gitlab::Shell::Error => e + error(:project, "https://github.com/#{repo}.git", e.message) + raise Github::RepositoryFetchError + end + end + + def fetch_wiki_repository + wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git" + wiki_path = "#{project.path_with_namespace}.wiki" + + unless project.wiki.repository_exists? + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors(:wiki, wiki_url, e.message) + end + end + + def fetch_labels + url = "/repos/#{repo}/labels" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + begin + representation = Github::Representation::Label.new(raw) + + label = project.labels.find_or_create_by!(title: representation.title) do |label| + label.color = representation.color + end + + cached[:label_ids][label.title] = label.id + rescue => e + error(:label, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_milestones + url = "/repos/#{repo}/milestones" + + while url + response = Github::Client.new(options).get(url, state: :all) + + response.body.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + next if project.milestones.where(iid: milestone.iid).exists? + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_pull_requests + url = "/repos/#{repo}/pulls" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) + next unless merge_request.new_record? && pull_request.valid? + + begin + restore_branches(pull_request) + + author_id = user_id(pull_request.author, project.creator_id) + description = format_description(pull_request.description, pull_request.author) + + merge_request.attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: pull_request.source_project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: pull_request.target_project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + milestone_id: milestone_id(pull_request.milestone), + author_id: author_id, + assignee_id: user_id(pull_request.assignee), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + merge_request.save!(validate: false) + merge_request.merge_request_diffs.create + + # Fetch review comments + review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" + fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) + + # Fetch comments + comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" + fetch_comments(merge_request, :comment, comments_url) + rescue => e + error(:pull_request, pull_request.url, e.message) + ensure + clean_up_restored_branches(pull_request) + end + end + + url = response.rels[:next] + end + end + + def fetch_issues + url = "/repos/#{repo}/issues" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + next unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) + end + end + rescue => e + error(:issue, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_comments(noteable, type, url, klass = Note) + while url + comments = Github::Client.new(options).get(url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + representation = Github::Representation::Comment.new(raw, options) + author_id = user_id(representation.author, project.creator_id) + + note = klass.new + note.project_id = project.id + note.noteable = noteable + note.note = format_description(representation.note, representation.author) + note.commit_id = representation.commit_id + note.line_code = representation.line_code + note.author_id = author_id + note.created_at = representation.created_at + note.updated_at = representation.updated_at + note.save!(validate: false) + rescue => e + error(type, representation.url, e.message) + end + end + end + + url = comments.rels[:next] + end + end + + def fetch_releases + url = "/repos/#{repo}/releases" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + representation = Github::Representation::Release.new(raw) + next unless representation.valid? + + release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) + next unless relese.new_record? + + begin + release.description = representation.description + release.created_at = representation.created_at + release.updated_at = representation.updated_at + release.save!(validate: false) + rescue => e + error(:release, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def restore_branches(pull_request) + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + end + + def restore_source_branch(pull_request) + repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def label_ids(labels) + labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + end + + def milestone_id(milestone) + return unless milestone.present? + + project.milestones.select(:id).find_by(iid: milestone.iid)&.id + end + + def user_id(user, fallback_id = nil) + return unless user.present? + return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) + + gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) + + cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? + cached[:user_ids][user.id] = gitlab_user_id || fallback_id + end + + def user_id_by_email(email) + return nil unless email + + ::User.find_by_any_email(email)&.id + end + + def user_id_by_external_uid(id) + return nil unless id + + ::User.select(:id) + .joins(:identities) + .merge(::Identity.where(provider: :github, extern_uid: id)) + .first&.id + end + + def format_description(body, author) + return body if cached[:gitlab_user_ids][author.id] + + "*Created by: #{author.username}*\n\n#{body}" + end + + def expire_repository_cache + repository.expire_content_cache + end + + def keep_track_of_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def error(type, url, message) + errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } + end + end +end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb new file mode 100644 index 00000000000..884693d093c --- /dev/null +++ b/lib/github/rate_limit.rb @@ -0,0 +1,27 @@ +module Github + class RateLimit + SAFE_REMAINING_REQUESTS = 100 + SAFE_RESET_TIME = 500 + RATE_LIMIT_URL = '/rate_limit'.freeze + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def get + response = connection.get(RATE_LIMIT_URL) + + # GitHub Rate Limit API returns 404 when the rate limit is disabled + return false unless response.status != 404 + + body = Oj.load(response.body, class_cache: false, mode: :compat) + remaining = body.dig('rate', 'remaining').to_i + reset_in = body.dig('rate', 'reset').to_i + exceed = remaining <= SAFE_REMAINING_REQUESTS + + [exceed, reset_in] + end + end +end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb new file mode 100644 index 00000000000..c1c9448f305 --- /dev/null +++ b/lib/github/repositories.rb @@ -0,0 +1,19 @@ +module Github + class Repositories + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch + Collection.new(options).fetch(repos_url) + end + + private + + def repos_url + '/user/repos' + end + end +end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb new file mode 100644 index 00000000000..f26bdbdd546 --- /dev/null +++ b/lib/github/representation/base.rb @@ -0,0 +1,30 @@ +module Github + module Representation + class Base + def initialize(raw, options = {}) + @raw = raw + @options = options + end + + def id + raw['id'] + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + private + + attr_reader :raw, :options + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb new file mode 100644 index 00000000000..d1dac6944f0 --- /dev/null +++ b/lib/github/representation/branch.rb @@ -0,0 +1,51 @@ +module Github + module Representation + class Branch < Representation::Base + attr_reader :repository + + def user + raw.dig('user', 'login') || 'unknown' + end + + def repo + return @repo if defined?(@repo) + + @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + end + + def ref + raw['ref'] + end + + def sha + raw['sha'] + end + + def short_sha + Commit.truncate_sha(sha) + end + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + private + + def branch_exists? + repository.branch_exists?(ref) + end + + def commit_exists? + repository.branch_names_contains(sha).include?(ref) + end + + def repository + @repository ||= options.fetch(:repository) + end + end + end +end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb new file mode 100644 index 00000000000..1b5be91461b --- /dev/null +++ b/lib/github/representation/comment.rb @@ -0,0 +1,42 @@ +module Github + module Representation + class Comment < Representation::Base + def note + raw['body'] || '' + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def commit_id + raw['commit_id'] + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + private + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw['diff_hunk'] + end + + def file_path + raw['path'] + end + end + end +end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb new file mode 100644 index 00000000000..9713b82615d --- /dev/null +++ b/lib/github/representation/issuable.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Issuable < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee'], options) + end + + def assigned? + raw['assignee'].present? + end + end + end +end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb new file mode 100644 index 00000000000..df3540a6e6c --- /dev/null +++ b/lib/github/representation/issue.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Issue < Representation::Issuable + def labels + raw['labels'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'opened' + end + + def has_comments? + raw['comments'] > 0 + end + + def has_labels? + labels.count > 0 + end + + def pull_request? + raw['pull_request'].present? + end + end + end +end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb new file mode 100644 index 00000000000..60aa51f9569 --- /dev/null +++ b/lib/github/representation/label.rb @@ -0,0 +1,13 @@ +module Github + module Representation + class Label < Representation::Base + def color + "##{raw['color']}" + end + + def title + raw['name'] + end + end + end +end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb new file mode 100644 index 00000000000..917e6394ad4 --- /dev/null +++ b/lib/github/representation/milestone.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Milestone < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['description'] + end + + def due_date + raw['due_on'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'active' + end + end + end +end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb new file mode 100644 index 00000000000..ac9c8283b4b --- /dev/null +++ b/lib/github/representation/pull_request.rb @@ -0,0 +1,78 @@ +module Github + module Representation + class PullRequest < Representation::Issuable + attr_reader :project + + delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + + def source_project + project + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def target_project + project + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def state + return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? + return 'closed' if raw['state'] == 'closed' + + 'opened' + end + + def opened? + state == 'opened' + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + private + + def project + @project ||= options.fetch(:project) + end + + def source_branch + @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" + end + + def target_branch + @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + end + end +end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb new file mode 100644 index 00000000000..e7e4b428c1a --- /dev/null +++ b/lib/github/representation/release.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Release < Representation::Base + def description + raw['body'] + end + + def tag + raw['tag_name'] + end + + def valid? + !raw['draft'] + end + end + end +end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb new file mode 100644 index 00000000000..6938aa7db05 --- /dev/null +++ b/lib/github/representation/repo.rb @@ -0,0 +1,6 @@ +module Github + module Representation + class Repo < Representation::Base + end + end +end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb new file mode 100644 index 00000000000..18591380e25 --- /dev/null +++ b/lib/github/representation/user.rb @@ -0,0 +1,15 @@ +module Github + module Representation + class User < Representation::Base + def email + return @email if defined?(@email) + + @email = Github::User.new(username, options).get.fetch('email', nil) + end + + def username + raw['login'] + end + end + end +end diff --git a/lib/github/response.rb b/lib/github/response.rb new file mode 100644 index 00000000000..761c524b553 --- /dev/null +++ b/lib/github/response.rb @@ -0,0 +1,25 @@ +module Github + class Response + attr_reader :raw, :headers, :status + + def initialize(response) + @raw = response + @headers = response.headers + @status = response.status + end + + def body + Oj.load(raw.body, class_cache: false, mode: :compat) + end + + def rels + links = headers['Link'].to_s.split(', ').map do |link| + href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures + + [name.to_sym, href] + end + + Hash[*links.flatten] + end + end +end diff --git a/lib/github/user.rb b/lib/github/user.rb new file mode 100644 index 00000000000..f88a29e590b --- /dev/null +++ b/lib/github/user.rb @@ -0,0 +1,24 @@ +module Github + class User + attr_reader :username, :options + + def initialize(username, options) + @username = username + @options = options + end + + def get + client.get(user_url).body + end + + private + + def client + @client ||= Github::Client.new(options) + end + + def user_url + "/users/#{username}" + end + end +end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 8c28009b9c6..4714ab18cc1 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -32,7 +32,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Master" => MASTER, + "Master" => MASTER } end @@ -47,7 +47,7 @@ module Gitlab guest: GUEST, reporter: REPORTER, developer: DEVELOPER, - master: MASTER, + master: MASTER } end @@ -60,7 +60,7 @@ module Gitlab "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, + "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL } end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d575367d81a..96d38f6daa0 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -14,30 +14,18 @@ module Gitlab # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref - # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter # - def self.render(input, context, asciidoc_opts = {}) - asciidoc_opts.reverse_merge!( - safe: :secure, - backend: :gitlab_html5, - attributes: [] - ) - asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + def self.render(input, context) + asciidoc_opts = { safe: :secure, + backend: :gitlab_html5, + attributes: DEFAULT_ADOC_ATTRS } + + context[:pipeline] = :markup plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) - - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..099c45dcfb7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,6 +37,9 @@ module Gitlab end def find_with_user_password(login, password) + # Avoid resource intensive login checks if password is not provided + return unless password.present? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -44,7 +47,7 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? + return unless Gitlab::LDAP::Config.enabled? Gitlab::LDAP::Authentication.login(login, password) else @@ -108,7 +111,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4efa20374a..5a6d9ae99a0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,7 +149,7 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description - merge_request = project.merge_requests.create( + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, @@ -168,7 +168,7 @@ module Gitlab import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue StandardError => e - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } end end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb new file mode 100644 index 00000000000..4fc9a075edc --- /dev/null +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -0,0 +1,138 @@ +# This class is not backed by a table in the main database. +# It loads the latest Pipeline for the HEAD of a repository, and caches that +# in Redis. +module Gitlab + module Cache + module Ci + class ProjectPipelineStatus + attr_accessor :sha, :status, :ref, :project, :loaded + + delegate :commit, to: :project + + def self.load_for_project(project) + new(project).tap do |status| + status.load_status + end + end + + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + + def self.update_for_pipeline(pipeline) + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed + end + + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) + @project = project + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache + end + + def has_status? + loaded? && sha.present? && status.present? + end + + def load_status + return if loaded? + + if has_cache? + load_from_cache + else + load_from_project + store_in_cache + end + + self.loaded = true + end + + def load_from_project + return unless commit + + self.sha = commit.sha + self.status = commit.status + self.ref = project.default_branch + end + + # We only cache the status for the HEAD commit of a project + # This status is rendered in project lists + def store_in_cache_if_needed + return delete_from_cache unless commit + return unless sha + return unless ref + + if commit.sha == sha && project.default_branch == ref + store_in_cache + end + end + + def load_from_cache + Gitlab::Redis.with do |redis| + self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + end + end + + def store_in_cache + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + end + end + + def delete_from_cache + Gitlab::Redis.with do |redis| + redis.del(cache_key) + end + end + + def has_cache? + return self.loaded unless self.loaded.nil? + + Gitlab::Redis.with do |redis| + redis.exists(cache_key) + end + end + + def loaded? + self.loaded + end + + def cache_key + self.class.cache_key_for_project(project) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index f34ed0f4cf2..3e0c30c33b7 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -5,7 +5,7 @@ module Gitlab Gitlab::ChatCommands::IssueShow, Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy, + Gitlab::ChatCommands::Deploy ].freeze def execute diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/chat_commands/presenters/issue_base.rb index 054f7f4be0c..25bc82994ba 100644 --- a/lib/gitlab/chat_commands/presenters/issue_base.rb +++ b/lib/gitlab/chat_commands/presenters/issue_base.rb @@ -22,7 +22,7 @@ module Gitlab [ { title: "Assignee", - value: @resource.assignee ? @resource.assignee.name : "_None_", + value: @resource.assignees.any? ? @resource.assignees.first.name : "_None_", short: true }, { diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index c85f79127bc..c984eb20606 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,24 +1,25 @@ module Gitlab module Checks class ChangeAccess - # protocol is currently used only in EE attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( - change, user_access:, project:, env: {}, skip_authorization: false, + change, user_access:, project:, skip_authorization: false, protocol: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) + @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @env = env @skip_authorization = skip_authorization @protocol = protocol end def exec - error = push_checks || tag_checks || protected_branch_checks + return GitAccessStatus.new(true) if skip_authorization + + error = push_checks || branch_checks || tag_checks if error GitAccessStatus.new(false, error) @@ -29,58 +30,95 @@ module Gitlab protected - def protected_branch_checks - return if skip_authorization + def push_checks + if user_access.cannot_do_action?(:push_code) + "You are not allowed to push code to this project." + end + end + + def branch_checks return unless @branch_name - return unless project.protected_branch?(@branch_name) + + if deletion? && @branch_name == project.default_branch + return "The default branch of a project cannot be deleted." + end + + protected_branch_checks + end + + def protected_branch_checks + return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) - return "You are not allowed to delete protected branches from this project." end + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + unless user_access.can_delete_branch?(@branch_name) + return 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.' + end + + unless protocol == 'web' + 'You can only delete protected branches using the web interface.' + end + end + + def protected_branch_push_checks if matching_merge_request? - if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) "You are not allowed to merge code into protected branches on this project." end else - if user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_push_to_branch?(@branch_name) "You are not allowed to push code to protected branches on this project." end end end def tag_checks - return if skip_authorization + return unless @tag_name - tag_ref = Gitlab::Git.tag_name(@ref) - - if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) - "You are not allowed to change existing tags on this project." + if tag_exists? && user_access.cannot_do_action?(:admin_project) + return "You are not allowed to change existing tags on this project." end + + protected_tag_checks end - def push_checks - return if skip_authorization + def protected_tag_checks + return unless ProtectedTag.protected?(project, @tag_name) - if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." + return "Protected tags cannot be updated." if update? + return "Protected tags cannot be deleted." if deletion? + + unless user_access.can_create_tag?(@tag_name) + return "You are not allowed to create this tag as it is protected." end end private - def protected_tag?(tag_name) - project.repository.tag_exists?(tag_name) + def tag_exists? + project.repository.tag_exists?(@tag_name) end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def update? + !Gitlab::Git.blank_ref?(@oldrev) && !deletion? + end + + def deletion? + Gitlab::Git.blank_ref?(@newrev) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index de0c9049ebf..1e73f89158d 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,20 +1,16 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev, env: {}) + def self.force_push?(project, oldrev, newrev) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute - - if exit_status == 0 - missed_ref.present? - else - raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." - end + Gitlab::Git::RevList.new( + path_to_repo: project.repository.path_to_repo, + oldrev: oldrev, newrev: newrev).missed_ref.present? end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 6f799c2f031..2e073334abc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -37,6 +37,12 @@ module Gitlab !directory? end + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + def has_parent? nodes > 0 end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 9b9a0a8125a..a78a85397bd 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -21,7 +21,13 @@ module Gitlab def validate_variables(variables) variables.is_a?(Hash) && - variables.all? { |key, value| validate_string(key) && validate_string(value) } + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) end def validate_string(value) diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c3b0e651c3a..8acab605c91 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -15,6 +15,10 @@ module Gitlab def self.default {} end + + def value + Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + end end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb new file mode 100644 index 00000000000..551483d0aaa --- /dev/null +++ b/lib/gitlab/ci/cron_parser.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + class CronParser + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze + VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze + + def initialize(cron, cron_timezone = 'UTC') + @cron = cron + @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name + end + + def next_time_from(time) + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? + end + + def cron_valid? + try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + end + + def cron_timezone_valid? + try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? + end + + private + + # NOTE: + # cron_timezone can only accept timezones listed in TZInfo::Timezone. + # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, + # because Rufus::Scheduler only supports TZInfo::Timezone. + # + # For example, those codes have the same effect. + # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) + # Time.zone = 'America/Los_Angeles' (TZInfo::Timezone) + # + # However, try_parse_cron only accepts the latter format. + # try_parse_cron('* * * * *', 'Pacific Time (US & Canada)') -> Doesn't work + # try_parse_cron('* * * * *', 'America/Los_Angeles') -> Works + # If you want to know more, please take a look + # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb + def try_parse_cron(cron, cron_timezone) + cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) + rescue + # noop + end + end + end +end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb new file mode 100644 index 00000000000..45fd0d4aa07 --- /dev/null +++ b/lib/gitlab/ci/status/build/action.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Action < Status::Extended + def label + if has_action? + @status.label + else + "#{@status.label} (not allowed)" + end + end + + def self.matches?(build, user) + build.action? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 67bbc3c4849..57b533bad99 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Cancelable < SimpleDelegator - include Status::Extended - + class Cancelable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 38ac6edc9f1..c852d607373 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -8,7 +8,8 @@ module Gitlab Status::Build::Retryable], [Status::Build::FailedAllowed, Status::Build::Play, - Status::Build::Stop]] + Status::Build::Stop], + [Status::Build::Action]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index 807afe24bd5..e42d3574357 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class FailedAllowed < SimpleDelegator - include Status::Extended - + class FailedAllowed < Status::Extended def label 'failed (allowed to fail)' end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 3495b8d0448..c6139f1b716 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Play < SimpleDelegator - include Status::Extended - + class Play < Status::Extended def label 'manual play action' end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 6b362af7634..505f80848b2 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Retryable < SimpleDelegator - include Status::Extended - + class Retryable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index e8530f2aaae..0b5199e5483 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Stop < SimpleDelegator - include Status::Extended - + class Stop < Status::Extended def label 'manual stop action' end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index d367c9bda69..1e8101f8949 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,13 +1,13 @@ module Gitlab module Ci module Status - module Extended - extend ActiveSupport::Concern + class Extended < SimpleDelegator + def initialize(status) + super(@status = status) + end - class_methods do - def matches?(_subject, _user) - raise NotImplementedError - end + def self.matches?(_subject, _user) + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb new file mode 100644 index 00000000000..cfd4329a923 --- /dev/null +++ b/lib/gitlab/ci/status/group/common.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Group + module Common + def has_details? + false + end + + def details_path + nil + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb new file mode 100644 index 00000000000..d118116cfc3 --- /dev/null +++ b/lib/gitlab/ci/status/group/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module Group + class Factory < Status::Factory + def self.common_helpers + Status::Group::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index a250c3fcb41..37dfe43fb62 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Pipeline - class Blocked < SimpleDelegator - include Status::Extended - + class Blocked < Status::Extended def text 'blocked' end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index d4cdab6957a..df6e76b0151 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -5,9 +5,7 @@ module Gitlab # Extended status used when pipeline or stage passed conditionally. # This means that failed jobs that are allowed to fail were present. # - class SuccessWarning < SimpleDelegator - include Status::Extended - + class SuccessWarning < Status::Extended def text 'passed' end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb new file mode 100644 index 00000000000..5b835bb669a --- /dev/null +++ b/lib/gitlab/ci/trace.rb @@ -0,0 +1,136 @@ +module Gitlab + module Ci + class Trace + attr_reader :job + + delegate :old_trace, to: :job + + def initialize(job) + @job = job + end + + def html(last_lines: nil) + read do |stream| + stream.html(last_lines: last_lines) + end + end + + def raw(last_lines: nil) + read do |stream| + stream.raw(last_lines: last_lines) + end + end + + def extract_coverage(regex) + read do |stream| + stream.extract_coverage(regex) + end + end + + def set(data) + write do |stream| + data = job.hide_secrets(data) + stream.set(data) + end + end + + def append(data, offset) + write do |stream| + current_length = stream.size + return -current_length unless current_length == offset + + data = job.hide_secrets(data) + stream.append(data, offset) + stream.size + end + end + + def exist? + current_path.present? || old_trace.present? + end + + def read + stream = Gitlab::Ci::Trace::Stream.new do + if current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + + def write + stream = Gitlab::Ci::Trace::Stream.new do + File.open(ensure_path, "a+b") + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + + def erase! + paths.each do |trace_path| + FileUtils.rm(trace_path, force: true) + end + + job.erase_old_trace! + end + + private + + def ensure_path + return current_path if current_path + + ensure_directory + default_path + end + + def ensure_directory + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) + end + end + + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + def paths + [ + default_path, + deprecated_path + ].compact + end + + def default_directory + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s + ) + end + + def default_path + File.join(default_directory, "#{job.id}.log") + end + + def deprecated_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project.ci_id.to_s, + "#{job.id}.log" + ) if job.project&.ci_id + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb new file mode 100644 index 00000000000..fa462cbe095 --- /dev/null +++ b/lib/gitlab/ci/trace/stream.rb @@ -0,0 +1,121 @@ +module Gitlab + module Ci + class Trace + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class Stream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 500.kilobytes + + attr_reader :stream + + delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + delegate :valid?, to: :stream, as: :present?, allow_nil: true + + def initialize + @stream = yield + @stream&.binmode + end + + def valid? + self.stream.present? + end + + def file? + self.path.present? + end + + def limit(last_bytes = LIMIT_SIZE) + if last_bytes < size + stream.seek(-last_bytes, IO::SEEK_END) + stream.readline + end + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def raw(last_lines: nil) + return unless valid? + + if last_lines.to_i > 0 + read_last_lines(last_lines) + else + stream.read + end.force_encoding(Encoding.default_external) + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html(last_lines: nil) + text = raw(last_lines: last_lines) + buffer = StringIO.new(text) + ::Ci::Ansi2html.convert(buffer).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex) + next unless matches.is_a?(Array) + next if matches.empty? + + match = matches.flatten.last + coverage = match.gsub(/\d+(\.\d+)?/).first + return coverage if coverage.present? + end + + nil + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + + private + + def read_last_lines(last_lines) + chunks = [] + pos = lines = 0 + max = stream.size + + # We want an extra line to make sure fist line has full contents + while lines <= last_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(last_lines).join + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb deleted file mode 100644 index 1d7ddeb3e0f..00000000000 --- a/lib/gitlab/ci/trace_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Ci - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 - class TraceReader - BUFFER_SIZE = 4096 - - attr_accessor :path, :buffer_size - - def initialize(new_path, buffer_size: BUFFER_SIZE) - self.path = new_path - self.buffer_size = Integer(buffer_size) - end - - def read(last_lines: nil) - if last_lines - read_last_lines(last_lines) - else - File.read(path) - end - end - - def read_last_lines(max_lines) - File.open(path) do |file| - chunks = [] - pos = lines = 0 - max = file.size - - # We want an extra line to make sure fist line has full contents - while lines <= max_lines && pos < max - pos += buffer_size - - buf = if pos <= max - file.seek(-pos, IO::SEEK_END) - file.read(buffer_size) - else # Reached the head, read only left - file.seek(0) - file.read(buffer_size - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) - end - - chunks.join.lines.last(max_lines).join - .force_encoding(Encoding.default_external) - end - end - end - end -end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 990b719ecfd..6e73361cad1 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -3,16 +3,33 @@ module Gitlab class FileCollection ConflictSideMissing = Class.new(StandardError) - attr_reader :merge_request, :our_commit, :their_commit + attr_reader :merge_request, :our_commit, :their_commit, :project - def initialize(merge_request) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit - end + delegate :repository, to: :project + + class << self + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def for_resolution(merge_request) + project = merge_request.source_project + + new(merge_request, project).tap do |file_collection| + project. + repository. + with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + + yield file_collection + end + end + end - def repository - merge_request.project.repository + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + def read_only(merge_request) + new(merge_request, merge_request.target_project) + end end def merge_index @@ -55,6 +72,15 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end + + private + + def initialize(merge_request, project) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + @project = project + end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 559e3939da6..cac31ea8cff 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -17,7 +17,7 @@ module Gitlab end def title - name.to_s.capitalize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def median diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 1e52b6614a1..5f9dc9a4303 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -13,12 +13,16 @@ module Gitlab :code end + def title + s_('CycleAnalyticsStage|Code') + end + def legend - "Related Merge Requests" + _("Related Merge Requests") end def description - "Time until first merge request" + _("Time until first merge request") end end end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 213994988a5..7b03811efb2 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -14,12 +14,16 @@ module Gitlab :issue end + def title + s_('CycleAnalyticsStage|Issue') + end + def legend - "Related Issues" + _("Related Issues") end def description - "Time before an issue gets scheduled" + _("Time before an issue gets scheduled") end end end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index bef3b95ff1b..1e11e84a9cb 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -7,7 +7,7 @@ module Gitlab test: :read_build, review: :read_merge_request, staging: :read_build, - production: :read_issue, + production: :read_issue }.freeze def self.get(*args) diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 45d51d30ccc..1a0afb56b4f 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -14,12 +14,16 @@ module Gitlab :plan end + def title + s_('CycleAnalyticsStage|Plan') + end + def legend - "Related Commits" + _("Related Commits") end def description - "Time before an issue starts implementation" + _("Time before an issue starts implementation") end end end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 9f387a02945..0fa8a65cb99 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -15,12 +15,16 @@ module Gitlab :production end + def title + s_('CycleAnalyticsStage|Production') + end + def legend - "Related Issues" + _("Related Issues") end def description - "From issue creation until deploy to production" + _("From issue creation until deploy to production") end def query diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index 4744be834de..cfbbdc43fd9 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -13,12 +13,16 @@ module Gitlab :review end + def title + s_('CycleAnalyticsStage|Review') + end + def legend - "Relative Merged Requests" + _("Related Merged Requests") end def description - "Time between merge request creation and merge/close" + _("Time between merge request creation and merge/close") end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index 3cdbe04fbaf..d5684bb9201 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -14,12 +14,16 @@ module Gitlab :staging end + def title + s_('CycleAnalyticsStage|Staging') + end + def legend - "Relative Deployed Builds" + _("Related Deployed Jobs") end def description - "From merge request merge until deploy to production" + _("From merge request merge until deploy to production") end end end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 43fa3795e5c..a917ddccac7 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -8,7 +8,7 @@ module Gitlab end def title - self.class.name.demodulize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def value diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 7b8faa4d854..bea78862757 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Commit < Base + def title + n_('Commit', 'Commits', value) + end + def value @value ||= count_commits end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 06032e9200e..099d798aac6 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Deploy < Base + def title + n_('Deploy', 'Deploys', value) + end + def value @value ||= @project.deployments.where("created_at > ?", @from).count end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 008468f24b9..9bbf7a2685f 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -9,7 +9,7 @@ module Gitlab end def title - 'New Issue' + n_('New Issue', 'New Issues', value) end def value diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index e96943833bc..2b5f72bef89 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -13,12 +13,16 @@ module Gitlab :test end + def title + s_('CycleAnalyticsStage|Test') + end + def legend - "Relative Builds Trigger by Commits" + _("Related Jobs") end def description - "Total test time for all commits/merges" + _("Total test time for all commits/merges") end def stage_query diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index f78106f5b10..8e74e18a311 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -36,7 +36,7 @@ module Gitlab user: { id: user.try(:id), name: user.try(:name), - email: user.try(:email), + email: user.try(:email) }, commit: { @@ -49,7 +49,7 @@ module Gitlab status: commit.status, duration: commit.duration, started_at: commit.started_at, - finished_at: commit.finished_at, + finished_at: commit.finished_at }, repository: { @@ -60,7 +60,7 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - }, + } } data diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index d76aa38f741..e81d19a7a2e 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -11,6 +11,7 @@ module Gitlab # ref: String, # user_id: String, # user_name: String, + # user_username: String, # user_email: String # project_id: String, # repository: { @@ -41,7 +42,7 @@ module Gitlab type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data - data = { + { object_kind: type, event_name: type, before: oldrev, @@ -51,6 +52,7 @@ module Gitlab message: message, user_id: user.id, user_name: user.name, + user_username: user.username, user_email: user.email, user_avatar: user.avatar_url, project_id: project.id, @@ -61,16 +63,15 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } - - data end # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - build(project, user, commits.last.id, commits.first.id, ref, commits) + commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + + build(project, user, commits.last&.id, commits.first&.id, ref, commits) end def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb new file mode 100644 index 00000000000..b42dc052949 --- /dev/null +++ b/lib/gitlab/data_builder/repository.rb @@ -0,0 +1,35 @@ +module Gitlab + module DataBuilder + module Repository + extend self + + # Produce a hash of post-receive data + def update(project, user, changes, refs) + { + event_name: 'repository_update', + + user_id: user.id, + user_name: user.name, + user_email: user.email, + user_avatar: user.avatar_url, + + project_id: project.id, + project: project.hook_attrs, + + changes: changes, + + refs: refs + } + end + + # Produce a hash of partial data for a single change + def single_change(oldrev, newrev, ref) + { + before: oldrev, + after: newrev, + ref: ref + } + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 63b8d0d3b9d..d0bd1299671 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -57,16 +57,16 @@ module Gitlab postgresql? ? "RANDOM()" : "RAND()" end - def true_value - if Gitlab::Database.postgresql? + def self.true_value + if postgresql? "'t'" else 1 end end - def false_value - if Gitlab::Database.postgresql? + def self.false_value + if postgresql? "'f'" else 0 diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fc445ab9483..e76c9abbe04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,6 +26,30 @@ module Gitlab add_index(table_name, column_name, options) end + # Removes an existed index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, :some_column + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ column: column_name })) + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When @@ -65,7 +89,8 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - ON DELETE #{on_delete} NOT VALID; + #{on_delete ? "ON DELETE #{on_delete}" : ''} + NOT VALID; EOF # Validate the existing constraint. This can potentially take a very @@ -90,6 +115,14 @@ module Gitlab execute('SET statement_timeout TO 0') if Database.postgresql? end + def true_value + Database.true_value + end + + def false_value + Database.false_value + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -226,6 +259,273 @@ module Gitlab raise error end end + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old - The old column name. + # new - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + def rename_column_concurrently(table, old, new, type: nil) + if transaction_open? + raise 'rename_column_concurrently can not be run inside a transaction' + end + + old_col = column_for(table, old) + new_type = type || old_col.type + + add_column(table, new, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new, old_col.default) if old_col.default + + trigger_name = rename_trigger_name(table, old, new) + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old) + quoted_new = quote_column_name(new) + + if Database.postgresql? + install_rename_triggers_for_postgresql(trigger_name, quoted_table, + quoted_old, quoted_new) + else + install_rename_triggers_for_mysql(trigger_name, quoted_table, + quoted_old, quoted_new) + end + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Changes the type of a column concurrently. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def change_column_type_concurrently(table, column, new_type) + temp_column = "#{column}_for_type_change" + + rename_column_concurrently(table, column, temp_column, type: new_type) + end + + # Performs cleanup of a concurrent type change. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def cleanup_concurrent_column_type_change(table, column) + temp_column = "#{column}_for_type_change" + + transaction do + # This has to be performed in a transaction as otherwise we might have + # inconsistent data. + cleanup_concurrent_column_rename(table, column, temp_column) + rename_column(table, temp_column, column) + end + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old - The name of the old column. + # new - The name of the new column. + def cleanup_concurrent_column_rename(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + if Database.postgresql? + remove_rename_triggers_for_postgresql(table, trigger_name) + else + remove_rename_triggers_for_mysql(trigger_name) + end + + remove_column(table, old) + end + + # Performs a concurrent column rename when using PostgreSQL. + def install_rename_triggers_for_postgresql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE OR REPLACE FUNCTION #{trigger}() + RETURNS trigger AS + $BODY$ + BEGIN + NEW.#{new} := NEW.#{old}; + RETURN NEW; + END; + $BODY$ + LANGUAGE 'plpgsql' + VOLATILE + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger} + BEFORE INSERT OR UPDATE + ON #{table} + FOR EACH ROW + EXECUTE PROCEDURE #{trigger}() + EOF + end + + # Installs the triggers necessary to perform a concurrent column rename on + # MySQL. + def install_rename_triggers_for_mysql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_insert + BEFORE INSERT + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_update + BEFORE UPDATE + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + end + + # Removes the triggers used for renaming a PostgreSQL column concurrently. + def remove_rename_triggers_for_postgresql(table, trigger) + execute("DROP TRIGGER #{trigger} ON #{table}") + execute("DROP FUNCTION #{trigger}()") + end + + # Removes the triggers used for renaming a MySQL column concurrently. + def remove_rename_triggers_for_mysql(trigger) + execute("DROP TRIGGER #{trigger}_insert") + execute("DROP TRIGGER #{trigger}_update") + end + + # Returns the (base) name to use for triggers when renaming columns. + def rename_trigger_name(table, old, new) + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) + end + + # Returns an Array containing the indexes for the given column + def indexes_for(table, column) + column = column.to_s + + indexes(table).select { |index| index.columns.include?(column) } + end + + # Returns an Array containing the foreign keys for the given column. + def foreign_keys_for(table, column) + column = column.to_s + + foreign_keys(table).select { |fk| fk.column == column } + end + + # Copies all indexes for the old column to a new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_indexes(table, old, new) + old = old.to_s + new = new.to_s + + indexes_for(table, old).each do |index| + new_columns = index.columns.map do |column| + column == old ? new : column + end + + # This is necessary as we can't properly rename indexes such as + # "ci_taggings_idx". + unless index.name.include?(old) + raise "The index #{index.name} can not be copied as it does not "\ + "mention the old column. You have to rename this index manually first." + end + + name = index.name.gsub(old, new) + + options = { + unique: index.unique, + name: name, + length: index.lengths, + order: index.orders + } + + # These options are not supported by MySQL, so we only add them if + # they were previously set. + options[:using] = index.using if index.using + options[:where] = index.where if index.where + + unless index.opclasses.blank? + opclasses = index.opclasses.dup + + # Copy the operator classes for the old column (if any) to the new + # column. + opclasses[new] = opclasses.delete(old) if opclasses[old] + + options[:opclasses] = opclasses + end + + add_concurrent_index(table, new_columns, options) + end + end + + # Copies all foreign keys for the old column to the new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_foreign_keys(table, old, new) + foreign_keys_for(table, old).each do |fk| + add_concurrent_foreign_key(fk.from_table, + fk.to_table, + column: new, + on_delete: fk.on_delete) + end + end + + # Returns the column for the given table and column name. + def column_for(table, name) + name = name.to_s + + columns(table).find { |column| column.name == name } + end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb new file mode 100644 index 00000000000..7ae5a4c17c8 --- /dev/null +++ b/lib/gitlab/database/multi_threaded_migration.rb @@ -0,0 +1,52 @@ +module Gitlab + module Database + module MultiThreadedMigration + MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection + + # This overwrites the default connection method so that every thread can + # use a thread-local connection, while still supporting all of Rails' + # migration methods. + def connection + Thread.current[MULTI_THREAD_AR_CONNECTION] || + ActiveRecord::Base.connection + end + + # Starts a thread-pool for N threads, along with N threads each using a + # single connection. The provided block is yielded from inside each + # thread. + # + # Example: + # + # with_multiple_threads(4) do + # execute('SELECT ...') + # end + # + # thread_count - The number of threads to start. + # + # join - When set to true this method will join the threads, blocking the + # caller until all threads have finished running. + # + # Returns an Array containing the started threads. + def with_multiple_threads(thread_count, join: true) + pool = Gitlab::Database.create_connection_pool(thread_count) + + threads = Array.new(thread_count) do + Thread.new do + pool.with_connection do |connection| + begin + Thread.current[MULTI_THREAD_AR_CONNECTION] = connection + yield + ensure + Thread.current[MULTI_THREAD_AR_CONNECTION] = nil + end + end + end + end + + threads.each(&:join) if join + + threads + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..5481024db8e --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,84 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + + def kind + type == 'Group' ? 'group' : 'user' + end + end + + class User < ActiveRecord::Base + self.table_name = 'users' + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..d60fd4bb551 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,132 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) + query.where(path_or_children) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..2958ad4b8e5 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,78 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + rename_user(old_full_path, new_full_path) if namespace.kind == 'user' + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def rename_user(old_username, new_username) + MigrationClasses::User.where(username: old_username) + .update_all(username: new_username) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb new file mode 100644 index 00000000000..c45ae8feb2c --- /dev/null +++ b/lib/gitlab/dependency_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + LINKERS = [ + GemfileLinker + ].freeze + + def self.linker(blob_name) + LINKERS.find { |linker| linker.support?(blob_name) } + end + + def self.link(blob_name, plain_text, highlighted_text) + linker = linker(blob_name) + return highlighted_text unless linker + + linker.link(plain_text, highlighted_text) + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb new file mode 100644 index 00000000000..5f4027e7e81 --- /dev/null +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -0,0 +1,109 @@ +module Gitlab + module DependencyLinker + class BaseLinker + class_attribute :file_type + + def self.support?(blob_name) + Gitlab::FileDetector.type_of(blob_name) == file_type + end + + def self.link(*args) + new(*args).link + end + + attr_accessor :plain_text, :highlighted_text + + def initialize(plain_text, highlighted_text) + @plain_text = plain_text + @highlighted_text = highlighted_text + end + + def link + link_dependencies + + highlighted_lines.join.html_safe + end + + private + + def package_url(name) + raise NotImplementedError + end + + def link_dependencies + raise NotImplementedError + end + + def package_link(name, url = package_url(name)) + return name unless url + + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + end + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call("gem") + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call("gem", "specific_package") + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call("github", /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call("name") + # # Will link `package` in `self.name = "package"` + def link_method_call(method_names, value = nil, &url_proc) + value = + case value + when String + Regexp.escape(value) + when nil + /[^'"]+/ + else + value + end + + method_names = Array(method_names).map { |name| Regexp.escape(name) } + + regex = %r{ + #{Regexp.union(method_names)} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + + # Links package names based on regex. + # + # Example: + # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) + # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` + def link_regex(regex) + highlighted_lines.map!.with_index do |rich_line, i| + marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + + marker.mark(regex, group: :name) do |text, left:, right:| + url = block_given? ? yield(text) : package_url(text) + package_link(text, url) + end + end + end + + def plain_lines + @plain_lines ||= plain_text.lines + end + + def highlighted_lines + @highlighted_lines ||= highlighted_text.lines + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb new file mode 100644 index 00000000000..9b82e126528 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -0,0 +1,29 @@ +module Gitlab + module DependencyLinker + class GemfileLinker < BaseLinker + self.file_type = :gemfile + + private + + def link_dependencies + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call("gem") + + # Link `github: "user/repo"` to https://github.com/user/repo + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| + "https://github.com/#{name}" + end + + # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + + # Link `source "https://rubygems.org"` to https://rubygems.org + link_method_call("source", %r{https?://[^'"]+}) { |url| url } + end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + end + end +end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 8406ca4269c..7948782aecc 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -18,6 +18,12 @@ module Gitlab head_sha == other.head_sha end + alias_method :eql?, :== + + def hash + [base_sha, start_sha, head_sha].hash + end + # There is only one case in which we will have `start_sha` and `head_sha`, # but not `base_sha`, which is when a diff is generated between an # orphaned branch and another branch, which means there _is_ no base, but diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb new file mode 100644 index 00000000000..c2a2eb15931 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -0,0 +1,17 @@ +module Gitlab + module Diff + class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + }.freeze + + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + symbol = MARKDOWN_SYMBOLS[mode] + "{#{symbol}#{text}#{symbol}}" + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 736933b1c4b..919965100ae 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,137 +1,21 @@ module Gitlab module Diff - class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - }.freeze - - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) - end - - def mark(line_inline_diffs, mode: nil, markdown: false) - return rich_line unless line_inline_diffs - - marker_ranges = [] - line_inline_diffs.each do |inline_diff_range| - # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range].flatten - # Turn the array of character positions into ranges - marker_ranges.concat(collapse_ranges(inline_diff_positions)) - end - - offset = 0 - - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "<span class='#{html_class_names(marker_ranges, mode, index)}'>" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "</span>" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + class InlineDiffMarker < Gitlab::StringRangeMarker + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - - rich_line.html_safe end private - def html_class_names(marker_ranges, mode, index) + def html_class_names(left, right, mode) class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 + class_names << "left" if left + class_names << "right" if right class_names << mode if mode class_names.join(" ") end - - # Mapping of character positions in the raw line, to the rich (highlighted) line - def position_mapping - @position_mapping ||= begin - mapping = [] - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - rich_char = rich_line[rich_pos] - - # The raw and rich lines are the same except for HTML tags, - # so skip over any `<...>` segment - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - # multi-char HTML entities in the rich line correspond to a single character in the raw line - if rich_char == '&' - multichar_mapping = [rich_pos] - until rich_char == ';' - rich_pos += 1 - multichar_mapping << rich_pos - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = multichar_mapping - else - mapping[raw_pos] = rich_pos - end - - rich_pos += 1 - end - - mapping - end - end - - # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - # Inserts tags around the characters identified by the given range - def insert_around_range(text, range, before, after, offset = 0) - # Just to be sure - return offset if offset + range.end + 1 > text.length - - text.insert(offset + range.begin, before) - offset += before.length - - text.insert(offset + range.end + 1, after) - offset += after.length - - offset - end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 114656958e3..0a15c6d9358 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -33,6 +33,10 @@ module Gitlab new_pos unless removed? || meta? end + def line + new_line || old_line + end + def unchanged? type.nil? end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..e89ff238ec7 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -16,7 +16,7 @@ module Gitlab end def trace(old_position) - return unless old_diff_refs.complete? && new_diff_refs.complete? + return unless old_diff_refs&.complete? && new_diff_refs&.complete? return unless old_position.diff_refs == old_diff_refs # Suppose we have an MR with source branch `feature` and target branch `master`. @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 496ee0bdcb0..38e27513281 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -131,10 +131,12 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", %W[git apply --check --3way #{patch_path}] ) do |output, status| + puts output unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') @@ -310,6 +312,17 @@ module Gitlab Resolve them, stage the changes and commit them. + If the patch couldn't be applied cleanly, use the following command: + + # In the EE repo + $ git apply --reject path/to/#{ce_branch}.patch + + This option makes git apply the parts of the patch that are applicable, + and leave the rejected hunks in corresponding `.rej` files. + You can then resolve the conflicts highlighted in `.rej` by + manually applying the correct diff from the `.rej` file to the file with conflicts. + When finished, you can delete the `.rej` files and commit your changes. + ⚠️ Don't forget to push your branch to gitlab-ee: # In the EE repo diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 32cece8316b..83440ae227d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -21,7 +21,7 @@ module Gitlab content_type: attachment.content_type } - link = ::Projects::UploadService.new(project, file).execute + link = UploadService.new(project, file).execute attachments << link if link ensure tmp.close! diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index 35ea2e0ef59..b07c68d1498 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -5,7 +5,11 @@ require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze + HANDLERS = [ + UnsubscribeHandler, + CreateNoteHandler, + CreateIssueHandler + ].freeze def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..a616a80e8f5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project&.full_path) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index d87ba427f4b..31579e94a87 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' @@ -8,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -27,32 +28,22 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project&.full_path) + end + private def author sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end def create_note - Notes::CreateService.new( - project, - author, - note: message, - noteable_type: sent_notification.noteable_type, - noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code, - position: sent_notification.position, - type: sent_notification.note_type - ).execute + sent_notification.create_reply(message) end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..5894384da5d 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end @@ -17,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project&.full_path) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..0d6b08b5d29 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,6 +31,8 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) + handler.execute end @@ -56,9 +57,8 @@ module Gitlab end def key_from_additional_headers(mail) - references = ensure_references_array(mail.references) - - find_key_from_references(references) + find_key_from_references(mail) || + find_key_from_delivered_to_header(mail) end def ensure_references_array(references) @@ -69,15 +69,24 @@ module Gitlab # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app Gitlab::IncomingEmail.scan_fallback_references(references) + when nil + [] end end - def find_key_from_references(references) - references.find do |mail_id| + def find_key_from_references(mail) + ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end + + def find_key_from_delivered_to_header(mail) + Array(mail[:delivered_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index a16d9fc2265..e3e36b35ce9 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -54,7 +54,7 @@ module Gitlab unicode_version: emoji_unicode_version(emoji_name) } - ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], data: data) + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data) end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index ffbc6e17dc5..270d67dd50c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -1,27 +1,23 @@ module Gitlab module EtagCaching class Middleware - RESERVED_WORDS = ProjectPathValidator::RESERVED.map { |word| "/#{word}/" }.join('|') - ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) - ) - def initialize(app) @app = app end def call(env) - return @app.call(env) unless enabled_for_current_route?(env) - Gitlab::Metrics.add_event(:etag_caching_middleware_used) + route = Gitlab::EtagCaching::Router.match(env) + return @app.call(env) unless route + + track_event(:etag_caching_middleware_used, route) etag, cached_value_present = get_etag(env) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag - Gitlab::Metrics.add_event(:etag_caching_cache_hit) - [304, { 'ETag' => etag }, ['']] + handle_cache_hit(etag, route) else - track_cache_miss(if_none_match, cached_value_present) + track_cache_miss(if_none_match, cached_value_present, route) status, headers, body = @app.call(env) headers['ETag'] = etag @@ -31,10 +27,6 @@ module Gitlab private - def enabled_for_current_route?(env) - ROUTE_REGEXP.match(env['PATH_INFO']) - end - def get_etag(env) cache_key = env['PATH_INFO'] store = Gitlab::EtagCaching::Store.new @@ -52,15 +44,27 @@ module Gitlab %Q{W/"#{value}"} end - def track_cache_miss(if_none_match, cached_value_present) + def handle_cache_hit(etag, route) + track_event(:etag_caching_cache_hit, route) + + status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 + + [status_code, { 'ETag' => etag }, []] + end + + def track_cache_miss(if_none_match, cached_value_present, route) if if_none_match.blank? - Gitlab::Metrics.add_event(:etag_caching_header_missing) + track_event(:etag_caching_header_missing, route) elsif !cached_value_present - Gitlab::Metrics.add_event(:etag_caching_key_not_found) + track_event(:etag_caching_key_not_found, route) else - Gitlab::Metrics.add_event(:etag_caching_resource_changed) + track_event(:etag_caching_resource_changed, route) end end + + def track_event(name, route) + Gitlab::Metrics.add_event(name, endpoint: route.name) + end end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb new file mode 100644 index 00000000000..ba31041d0c1 --- /dev/null +++ b/lib/gitlab/etag_caching/router.rb @@ -0,0 +1,51 @@ +module Gitlab + module EtagCaching + class Router + Route = Struct.new(:regexp, :name) + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) + ROUTES = [ + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), + 'issue_notes' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/realtime_changes\z), + 'issue_title' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), + 'commit_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), + 'new_merge_request_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), + 'merge_request_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), + 'project_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), + 'project_pipeline' + ) + ].freeze + + def self.match(env) + ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + end + end + end +end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index c9ca4cadd1c..a8cb7fc3fe7 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -5,15 +5,33 @@ module Gitlab # a README or a CONTRIBUTING file. module FileDetector PATTERNS = { + # Project files readme: /\Areadme/i, changelog: /\A(changelog|history|changes|news)/i, license: /\A(licen[sc]e|copying)(\..+|\z)/i, contributing: /\Acontributing/i, version: 'version', + avatar: /\Alogo\.(png|jpg|gif)\z/, + + # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - avatar: /\Alogo\.(png|jpg|gif)\z/ + route_map: 'route-map.yml', + + # Dependency files + cartfile: /\ACartfile/, + composer_json: 'composer.json', + gemfile: /\A(Gemfile|gems\.rb)\z/, + gemfile_lock: 'Gemfile.lock', + gemspec: /\.gemspec\z/, + godeps_json: 'Godeps.json', + package_json: 'package.json', + podfile: 'Podfile', + podspec_json: /\.podspec\.json\z/, + podspec: /\.podspec\z/, + requirements_txt: /requirements\.txt\z/, + yarn_lock: 'yarn.lock' }.freeze # Returns an Array of file types based on the given paths. diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb new file mode 100644 index 00000000000..093d9ed8092 --- /dev/null +++ b/lib/gitlab/file_finder.rb @@ -0,0 +1,32 @@ +# This class finds files in a repository by name and content +# the result is joined and sorted by file name +module Gitlab + class FileFinder + BATCH_SIZE = 100 + + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + end + + def find(query) + blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = Gitlab::ProjectSearchResults.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| + results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 222bcdcbf9c..3dcee681c72 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -122,15 +122,15 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - iid: bug['ixBug'], - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed', - created_at: date, - updated_at: DateTime.parse(bug['dtLastUpdated']) + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_ids: [assignee_id], + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d3df3f1bca1..936606152e9 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -4,6 +4,8 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze + CommandError = Class.new(StandardError) + class << self def ref_name(ref) ref.sub(/\Arefs\/(tags|heads)\//, '') diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e56eb0d3beb..c1b31618e0d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -8,7 +8,7 @@ module Gitlab # the user. We load as much as we can for encoding detection # (Linguist) and LFS pointer parsing. All other cases where we need full # blob data should use load_all_data!. - MAX_DATA_DISPLAY_SIZE = 10485760 + MAX_DATA_DISPLAY_SIZE = 10.megabytes attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary @@ -90,7 +90,7 @@ module Gitlab name: blob_entry[:name], data: '', path: path, - commit_id: sha, + commit_id: sha ) end end @@ -109,10 +109,6 @@ module Gitlab @binary.nil? ? super : @binary == true end - def empty? - !data || data == '' - end - def data encode! @data end @@ -132,6 +128,10 @@ module Gitlab encode! @name end + def truncated? + size && (size > loaded_size) + end + # Valid LFS object pointer is a text file consisting of # version # oid @@ -153,16 +153,20 @@ module Gitlab def lfs_size if has_lfs_version_key? size = data.match(/(?<=size )([0-9]+)/) - return size[1] if size + return size[1].to_i if size end nil end - def truncated? - size && (size > loaded_size) + def external_storage + return unless lfs_pointer? + + :lfs end + alias_method :external_size, :lfs_size + private def has_lfs_version_key? diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 586380da94a..124526e4b59 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,40 @@ module Gitlab module Git class Branch < Ref + def initialize(repository, name, target) + if target.is_a?(Gitaly::FindLocalBranchResponse) + target = target_from_gitaly_local_branches_response(target) + end + + super(repository, name, target) + end + + def target_from_gitaly_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name, + author_email: response.commit_author.email, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name, + committer_email: response.commit_committer.email + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 3a73697dc5d..297531db4cc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -19,13 +19,7 @@ module Gitlab def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) - methods = [:message, :parent_ids, :authored_date, :author_name, - :author_email, :committed_date, :committer_name, - :committer_email] - - methods.all? do |method| - send(method) == other.send(method) - end + id && id == other.id end class << self @@ -55,6 +49,7 @@ module Gitlab # Commit.find(repo, 'master') # def find(repo, commit_id = "HEAD") + return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) obj = if commit_id.is_a?(String) @@ -192,6 +187,10 @@ module Gitlab Commit.diff_from_parent(raw_commit, options) end + def deltas + @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + end + def has_zero_stats? stats.total.zero? rescue diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 019be151353..31d1b66b4f7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -183,6 +183,8 @@ module Gitlab when Gitaly::CommitDiffResponse init_from_gitaly(raw_diff) prune_diff_if_eligible(collapse) + when Gitaly::CommitDelta + init_from_gitaly(raw_diff) when nil raise "Nil as raw diff passed" else @@ -278,15 +280,15 @@ module Gitlab end end - def init_from_gitaly(diff_msg) - @diff = diff_msg.raw_chunks.join - @new_path = encode!(diff_msg.to_path.dup) - @old_path = encode!(diff_msg.from_path.dup) - @a_mode = diff_msg.old_mode.to_s(8) - @b_mode = diff_msg.new_mode.to_s(8) - @new_file = diff_msg.from_id == BLANK_SHA - @renamed_file = diff_msg.from_path != diff_msg.to_path - @deleted_file = diff_msg.to_id == BLANK_SHA + def init_from_gitaly(msg) + @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) + @new_path = encode!(msg.to_path.dup) + @old_path = encode!(msg.from_path.dup) + @a_mode = msg.old_mode.to_s(8) + @b_mode = msg.new_mode.to_s(8) + @new_file = msg.from_id == BLANK_SHA + @renamed_file = msg.from_path != msg.to_path + @deleted_file = msg.to_id == BLANK_SHA end def prune_diff_if_eligible(collapse = false) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 4e45ec7c174..bcbad8ec829 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -15,7 +15,6 @@ module Gitlab @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file @all_diffs = !!options.fetch(:all_diffs, false) @no_collapse = !!options.fetch(:no_collapse, true) - @deltas_only = !!options.fetch(:deltas_only, false) @line_count = 0 @byte_count = 0 @@ -27,8 +26,6 @@ module Gitlab if @populated # @iterator.each is slower than just iterating the array in place @array.each(&block) - elsif @deltas_only - each_delta(&block) else Gitlab::GitalyClient.migrate(:commit_raw_diffs) do each_patch(&block) @@ -81,14 +78,6 @@ module Gitlab files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes end - def each_delta - @iterator.each_delta.with_index do |delta, i| - diff = Gitlab::Git::Diff.new(delta) - - yield @array[i] = diff - end - end - def each_patch @iterator.each_with_index do |raw, i| # First yield cached Diff instances from @array diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb new file mode 100644 index 00000000000..0fdc57ec954 --- /dev/null +++ b/lib/gitlab/git/env.rb @@ -0,0 +1,38 @@ +module Gitlab + module Git + # Ephemeral (per request) storage for environment variables that some Git + # commands may need. + # + # For example, in pre-receive hooks, new objects are put in a temporary + # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved + # (this would break push rules for instance). + # + # This class is thread-safe via RequestStore. + class Env + WHITELISTED_GIT_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY + GIT_ALTERNATE_OBJECT_DIRECTORIES + ].freeze + + def self.set(env) + return unless RequestStore.active? + + RequestStore.store[:gitlab_git_env] = whitelist_git_env(env) + end + + def self.all + return {} unless RequestStore.active? + + RequestStore.fetch(:gitlab_git_env) { {} } + end + + def self.[](key) + all[key] + end + + def self.whitelist_git_env(env) + env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access + end + end + end +end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index af1744c9c46..1add037fa5f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,8 +1,12 @@ module Gitlab module Git class Index + IndexError = Class.new(StandardError) + DEFAULT_MODE = 0o100644 + ACTIONS = %w(create create_dir update move delete).freeze + attr_reader :repository, :raw_index def initialize(repository) @@ -23,9 +27,8 @@ module Gitlab def create(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end add_blob(options) @@ -34,13 +37,12 @@ module Gitlab def create_dir(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end if dir_exists?(options[:file_path]) - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + raise IndexError, "A directory with this name already exists" end options = options.dup @@ -55,7 +57,7 @@ module Gitlab file_entry = get(options[:file_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" end add_blob(options, mode: file_entry[:mode]) @@ -66,7 +68,11 @@ module Gitlab file_entry = get(options[:previous_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" + end + + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end raw_index.remove(options[:previous_path]) @@ -77,9 +83,8 @@ module Gitlab def delete(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + unless get(options[:file_path]) + raise IndexError, "A file with this name doesn't exist" end raw_index.remove(options[:file_path]) @@ -95,10 +100,20 @@ module Gitlab end def normalize_path(path) + unless path + raise IndexError, "You must provide a file path" + end + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - if pathname.each_filename.include?('..') - raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + pathname.each_filename do |segment| + if segment == '..' + raise IndexError, 'Path cannot include directory traversal' + end + + unless segment =~ Gitlab::Regex.file_name_regex + raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" + end end pathname.to_s @@ -106,6 +121,10 @@ module Gitlab def add_blob(options, mode: nil) content = options[:content] + unless content + raise IndexError, "You must provide content" + end + content = Base64.decode64(content) if options[:encoding] == 'base64' detect = CharlockHolmes::EncodingDetector.new.detect(content) @@ -119,7 +138,7 @@ module Gitlab raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) rescue Rugged::IndexError => e - raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + raise IndexError, e.message end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4e72519c81d..b9f1ac144b6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,10 @@ module Gitlab class Repository include Gitlab::Git::Popen + ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY + GIT_ALTERNATE_OBJECT_DIRECTORIES + ].freeze SEARCH_CONTEXT_LINES = 3 NoRepository = Class.new(StandardError) @@ -23,11 +27,17 @@ module Gitlab # Rugged repo object attr_reader :rugged + attr_reader :storage + # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(path) - @path = path - @name = path.split("/").last + def initialize(storage, relative_path) + @storage = storage + @relative_path = relative_path + + storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @path = File.join(storage_path, @relative_path) + @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) end @@ -37,7 +47,13 @@ module Gitlab # Default branch in the repository def root_ref - @root_ref ||= discover_default_branch + @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| + if is_enabled + gitaly_ref_client.default_branch_name + else + discover_default_branch + end + end end # Alias to old method for compatibility @@ -46,7 +62,7 @@ module Gitlab end def rugged - @rugged ||= Rugged::Repository.new(path) + @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories) rescue Rugged::RepositoryError, Rugged::OSError raise NoRepository.new('no repository for such path') end @@ -54,18 +70,26 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - branches.map(&:name) + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names + else + branches.map(&:name) + end + end end # Returns an Array of Branches - def branches - rugged.branches.map do |rugged_ref| + def branches(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| begin Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) rescue Rugged::ReferenceError # Omit invalid branch end - end.compact.sort_by(&:name) + end.compact + + sort_branches(branches, sort_by) end def reload_rugged @@ -86,28 +110,57 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref end - def local_branches - rugged.branches.each(:local).map do |branch| - Gitlab::Git::Branch.new(self, branch.name, branch.target) + def local_branches(sort_by: nil) + gitaly_migrate(:local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch| + Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch) + end + else + branches(filter: :local, sort_by: sort_by) + end end end # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end # Returns an Array of tag names def tag_names - rugged.tags.map { |t| t.name } + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names + else + rugged.tags.map { |t| t.name } + end + end end # Returns an Array of Tags @@ -215,7 +268,7 @@ module Gitlab 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(prefix, storage_path, format), - 'CommitId' => commit.id, + 'CommitId' => commit.id } end @@ -411,6 +464,11 @@ module Gitlab rugged.merge_base(from, to) end + # Returns true is +from+ is direct ancestor to +to+, otherwise false + def is_ancestor?(from, to) + gitaly_commit_client.is_ancestor(from, to) + end + # Return an array of Diff objects that represent the diff # between +from+ and +to+. See Diff::filter_diff_options for the allowed # diff options. The +options+ hash can also include :break_rewrites to @@ -419,6 +477,23 @@ module Gitlab Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) end + # Returns a RefName for a given SHA + def ref_name_for_sha(ref_path, sha) + raise ArgumentError, "sha can't be empty" unless sha.present? + + gitaly_migrate(:find_ref_name) do |is_enabled| + if is_enabled + gitaly_ref_client.find_ref_name(sha, ref_path) + else + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, @path).first.split.last + end + end + end + # Returns commits collection # # Ex. @@ -434,7 +509,10 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :date(default) or :topo + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) actual_options = options.dup @@ -462,11 +540,8 @@ module Gitlab end end - if actual_options[:order] == :topo - walker.sorting(Rugged::SORT_TOPO) - else - walker.sorting(Rugged::SORT_NONE) - end + sort_type = rugged_sort_type(actual_options[:order]) + walker.sorting(sort_type) commits = [] offset = actual_options[:skip] @@ -812,27 +887,6 @@ module Gitlab rugged.remotes[remote_name].push(refspecs) end - # Merge the +source_name+ branch into the +target_name+ branch. This is - # equivalent to `git merge --no_ff +source_name+`, since a merge commit - # is always created. - def merge(source_name, target_name, options = {}) - our_commit = rugged.branches[target_name].target - their_commit = rugged.branches[source_name].target - - raise "Invalid merge target" if our_commit.nil? - raise "Invalid merge source" if their_commit.nil? - - merge_index = rugged.merge_commits(our_commit, their_commit) - return false if merge_index.conflicts? - - actual_options = options.merge( - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - update_ref: "refs/heads/#{target_name}" - ) - Rugged::Commit.create(rugged, actual_options) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -920,8 +974,16 @@ module Gitlab @attributes.attributes(path) end + def gitaly_repository + Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + end + private + def alternate_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact + end + # Get the content of a blob for a given commit. If the blob is a commit # (for submodules) then return the blob's OID. def blob_content(commit, blob_name) @@ -1056,56 +1118,6 @@ module Gitlab end end - def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) - git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) - - # Put files into a directory before archiving - prefix = "#{archive_name(treeish)}/" - git_archive_cmd << "--prefix=#{prefix}" - - # Format defaults to tar - git_archive_cmd << "--format=#{format}" if format - - git_archive_cmd += %W(-- #{treeish}) - - open(filename, 'w') do |file| - # Create a pipe to act as the '|' in 'git archive ... | gzip' - pipe_rd, pipe_wr = IO.pipe - - # Get the compression process ready to accept data from the read end - # of the pipe - compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) - # The read end belongs to the compression process now; we should - # close our file descriptor for it. - pipe_rd.close - - # Start 'git archive' and tell it to write into the write end of the - # pipe. - git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) - # The write end belongs to 'git archive' now; close it. - pipe_wr.close - - # When 'git archive' and the compression process are finished, we are - # done. - Process.waitpid(git_archive_pid) - raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? - Process.waitpid(compress_pid) - raise "#{compress_cmd.join(' ')} failed" unless $?.success? - end - end - - def nice(cmd) - nice_cmd = %w(nice -n 20) - unless unsupported_platform? - nice_cmd += %w(ionice -c 2 -n 7) - end - nice_cmd + cmd - end - - def unsupported_platform? - %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? - end - # Returns true if the index entry has the special file mode that denotes # a submodule. def submodule?(index_entry) @@ -1197,6 +1209,53 @@ module Gitlab diff.find_similar!(break_rewrites: break_rewrites) diff.each_patch end + + def sort_branches(branches, sort_by) + case sort_by + when 'name' + branches.sort_by(&:name) + when 'updated_desc' + branches.sort do |a, b| + b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date + end + when 'updated_asc' + branches.sort do |a, b| + a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date + end + else + branches + end + end + + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE | Rugged::SORT_TOPO + } + + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 79dd0cf7df2..a16b0ed76f4 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,41 +1,42 @@ module Gitlab module Git class RevList - attr_reader :project, :env - - ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze - - def initialize(oldrev, newrev, project:, env: nil) - @project = project - @env = env.presence || {} - @args = [Gitlab.config.git.bin_path, - "--git-dir=#{project.repository.path_to_repo}", - "rev-list", - "--max-count=1", - oldrev, - "^#{newrev}"] + attr_reader :oldrev, :newrev, :path_to_repo + + def initialize(path_to_repo:, newrev:, oldrev: nil) + @oldrev = oldrev + @newrev = newrev + @path_to_repo = path_to_repo end - def execute - Gitlab::Popen.popen(@args, nil, parse_environment_variables) + # This method returns an array of new references + def new_refs + execute([*base_args, newrev, '--not', '--all']) end - def valid? - environment_variables.all? do |(name, value)| - value.to_s.start_with?(project.repository.path_to_repo) - end + # This methods returns an array of missed references + def missed_ref + execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end private - def parse_environment_variables - return {} unless valid? + def execute(args) + output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + + unless status.zero? + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + end - environment_variables + output.split("\n") end - def environment_variables - @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + def base_args + [ + Gitlab.config.git.bin_path, + "--git-dir=#{path_to_repo}", + 'rev-list' + ] end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index f7450e8b58f..d41256d9a84 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -33,9 +33,9 @@ module Gitlab root_id: root_tree.oid, name: entry[:name], type: entry[:type], - mode: entry[:filemode], + mode: entry[:filemode].to_s(8), path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha, + commit_id: sha ) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index eea2f206902..99724db8da2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -18,13 +18,12 @@ module Gitlab attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:, env: {}) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) - @env = env end def check(cmd, changes) @@ -152,7 +151,6 @@ module Gitlab change, user_access: user_access, project: project, - env: @env, skip_authorization: deploy_key?, protocol: protocol ).exec diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 6babea144c7..742118b76a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -1,43 +1,30 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :repo_path, :identifier, :changes, :project + attr_reader :project, :identifier, :changes - def initialize(repo_path, identifier, changes) - repo_path.gsub!(/\.git\z/, '') - repo_path.gsub!(/\A\//, '') - - @repo_path = repo_path + def initialize(project, identifier, changes) + @project = project @identifier = identifier @changes = deserialize_changes(changes) - - retrieve_project_and_type - end - - def wiki? - @type == :wiki - end - - def regular_project? - @type == :project end def identify(revision) super(identifier, project, revision) end - private + def changes_refs + return enum_for(:changes_refs) unless block_given? - def retrieve_project_and_type - @type = :project - @project = Project.find_by_full_path(@repo_path) + changes.each do |change| + oldrev, newrev, ref = change.strip.split(' ') - if @repo_path.end_with?('.wiki') && !@project - @type = :wiki - @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) + yield oldrev, newrev, ref end end + private + def deserialize_changes(changes) changes = utf8_encode_changes(changes) changes.lines diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a0dbe0a8c11..72466700c05 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,26 +4,42 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - def self.configure_channel(storage, address) - @addresses ||= {} - @addresses[storage] = address - @channels ||= {} - @channels[storage] = new_channel(address) - end + MUTEX = Mutex.new + private_constant :MUTEX - def self.new_channel(address) - # NOTE: Gitaly currently runs on a Unix socket, so permissions are - # handled using the file system and no additional authentication is - # required (therefore the :this_channel_is_insecure flag) - GRPC::Core::Channel.new(address, {}, :this_channel_is_insecure) + def self.stub(name, storage) + MUTEX.synchronize do + @stubs ||= {} + @stubs[storage] ||= {} + @stubs[storage][name] ||= begin + klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + addr = address(storage) + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + klass.new(addr, :this_channel_is_insecure) + end + end end - def self.get_channel(storage) - @channels[storage] + def self.clear_stubs! + MUTEX.synchronize do + @stubs = nil + end end - def self.get_address(storage) - @addresses[storage] + def self.address(storage) + params = Gitlab.config.repositories.storages[storage] + raise "storage not found: #{storage.inspect}" if params.nil? + + address = params['gitaly_address'] + unless address.present? + raise "storage #{storage.inspect} is missing a gitaly_address" + end + + unless URI(address).scheme.in?(%w(tcp unix)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + end + + address end def self.enabled? diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 9c714a3ee45..4491903d788 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,23 +5,55 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - class << self - def diff_from_parent(commit, options = {}) - project = commit.project - channel = GitalyClient.get_channel(project.repository_storage) - stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel) - repo = Gitaly::Repository.new(path: project.repository.path_to_repo) - parent = commit.parents[0] - parent_id = parent ? parent.id : EMPTY_TREE_ID - request = Gitaly::CommitDiffRequest.new( - repository: repo, - left_commit_id: parent_id, - right_commit_id: commit.id - ) - - Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + stub = GitalyClient.stub(:commit, @repository.storage) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + stub.commit_is_ancestor(request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) + Gitlab::Git::DiffCollection.new(response, options) + end + + def commit_deltas(commit) + request_params = commit_diff_request_params(commit) + + response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def diff_service_stub + GitalyClient.stub(:diff, @repository.storage) + end end end end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb index cbfb129c002..719554eac52 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -3,18 +3,14 @@ module Gitlab class Notifications attr_accessor :stub - def initialize(repo_path) - full_path = Gitlab::RepoPath.strip_storage_path(repo_path). - sub(/\.git\z/, '').sub(/\.wiki\z/, '') - @project = Project.find_by_full_path(full_path) - - channel = GitalyClient.get_channel(@project.repository_storage) - @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel) + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = GitalyClient.stub(:notifications, repository.storage) end def post_receive - repository = Gitaly::Repository.new(path: @project.repository.path_to_repo) - request = Gitaly::PostReceiveRequest.new(repository: repository) + request = Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) @stub.post_receive(request) end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb new file mode 100644 index 00000000000..227fe45642e --- /dev/null +++ b/lib/gitlab/gitaly_client/ref.rb @@ -0,0 +1,72 @@ +module Gitlab + module GitalyClient + class Ref + attr_accessor :stub + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = GitalyClient.stub(:ref, repository.storage) + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + branch_name = stub.find_default_branch_name(request).name + + Gitlab::Git.branch_name(branch_name) + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/') + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) + consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') + end + + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @gitaly_repo, + commit_id: commit_id, + prefix: ref_prefix + ) + + stub.find_ref_name(request).name + end + + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + consume_branches_response(stub.find_local_branches(request)) + end + + private + + def consume_refs_response(response, prefix:) + response.flat_map do |r| + r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } + end + end + + def sort_by_param(sort_by) + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map { |r| r.branches } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb new file mode 100644 index 00000000000..86d055d3533 --- /dev/null +++ b/lib/gitlab/gitaly_client/util.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + module Util + class << self + def repository(repository_storage, relative_path) + Gitaly::Repository.new( + path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), + storage_name: repository_storage, + relative_path: relative_path + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 5d29e698b27..8aa885fb811 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -11,6 +11,14 @@ module Gitlab sha.present? && ref.present? end + def user + raw_data.user&.login || 'unknown' + end + + def short_sha + Commit.truncate_sha(sha) + end + private def branch_exists? diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index eea4a91f17d..a8c0b47e786 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -157,7 +157,7 @@ module Gitlab end def restore_source_branch(pull_request) - project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name) + project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end def restore_target_branch(pull_request) diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 6f5ac4dac0d..977cd0423ba 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -10,7 +10,7 @@ module Gitlab description: description, state: state, author_id: author_id, - assignee_id: assignee_id, + assignee_ids: Array(assignee_id), created_at: raw_data.created_at, updated_at: raw_data.updated_at } diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index add7236e339..150afa31432 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,8 +1,8 @@ module Gitlab module GithubImport class PullRequestFormatter < IssuableFormatter - delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true + delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true def attributes { @@ -20,7 +20,8 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: raw_data.updated_at + updated_at: raw_data.updated_at, + imported: true } end @@ -37,13 +38,20 @@ module Gitlab end def source_branch_name - @source_branch_name ||= begin - if cross_project? - "pull/#{number}/#{source_branch_repo.full_name}/#{source_branch_ref}" + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed else - source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}" + source_branch_ref end - end + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}" + end + + def source_branch_exists? + !cross_project? && source_branch.exists? end def target_branch @@ -51,13 +59,17 @@ module Gitlab end def target_branch_name - @target_branch_name ||= begin - target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}" - end + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}" end def cross_project? - source_branch.repo.id != target_branch.repo.id + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id end def opened? diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb new file mode 100644 index 00000000000..07c0abcce23 --- /dev/null +++ b/lib/gitlab/gl_repository.rb @@ -0,0 +1,20 @@ +module Gitlab + module GlRepository + def self.gl_repository(project, is_wiki) + "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" + end + + def self.parse(gl_repository) + match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) + unless match_data + raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" + end + + type, id = match_data.captures + project = Project.find_by(id: id) + wiki = type == 'wiki' + + [project, wiki] + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5ab84266b7d..6200bd460ea 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -10,11 +10,14 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') + gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.gitlab_url = Gitlab.config.gitlab.url if current_user gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name + gon.current_user_avatar_url = current_user.avatar_url end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..1b43440673c 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -92,13 +108,13 @@ module Gitlab end issue = Issue.create!( - iid: raw_issue['id'], - project_id: project.id, - title: raw_issue['title'], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_ids: [assignee_id], + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb new file mode 100644 index 00000000000..7de6d4d9367 --- /dev/null +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -0,0 +1,45 @@ +module Gitlab + module HealthChecks + module BaseAbstractCheck + def name + super.demodulize.underscore + end + + def human_name + name.sub(/_check$/, '').capitalize + end + + def readiness + raise NotImplementedError + end + + def liveness + HealthChecks::Result.new(true) + end + + def metrics + [] + end + + protected + + def metric(name, value, **labels) + Metric.new(name, value, labels) + end + + def with_timing(proc) + start = Time.now + result = proc.call + yield result, Time.now.to_f - start.to_f + end + + def catch_timeout(seconds, &block) + begin + Timeout.timeout(seconds.to_i, &block) + rescue Timeout::Error => ex + ex + end + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb new file mode 100644 index 00000000000..fd94984f8a2 --- /dev/null +++ b/lib/gitlab/health_checks/db_check.rb @@ -0,0 +1,29 @@ +module Gitlab + module HealthChecks + class DbCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'db_ping' + end + + def is_successful?(result) + result == '1' + end + + def check + catch_timeout 10.seconds do + if Gitlab::Database.postgresql? + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + else + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb new file mode 100644 index 00000000000..df962d203b7 --- /dev/null +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -0,0 +1,117 @@ +module Gitlab + module HealthChecks + class FsShardsCheck + extend BaseAbstractCheck + + class << self + def readiness + repository_storages.map do |storage_name| + begin + tmp_file_path = tmp_file_path(storage_name) + + if !storage_stat_test(storage_name) + HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) + elsif !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + rescue RuntimeError => ex + message = "unexpected error #{ex} when checking storage #{storage_name}" + Rails.logger.error(message) + HealthChecks::Result.new(false, message, shard: storage_name) + ensure + delete_test_file(tmp_file_path) + end + end + end + + def metrics + repository_storages.flat_map do |storage_name| + tmp_file_path = tmp_file_path(storage_name) + [ + operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), + operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), + operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + ].flatten + end + end + + private + + RANDOM_STRING = SecureRandom.hex(1000).freeze + + def operation_metrics(ok_metric, latency_metric, operation, **labels) + with_timing operation do |result, elapsed| + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] + end + rescue RuntimeError => ex + Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + [metric(ok_metric, 0, **labels)] + end + + def repository_storages + @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages + end + + def storages_paths + @storage_paths ||= Gitlab.config.repositories.storages + end + + def with_timeout(args) + %w{timeout 1}.concat(args) + end + + def tmp_file_path(storage_name) + Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + end + + def path(storage_name) + storages_paths&.dig(storage_name, 'path') + end + + def storage_stat_test(storage_name) + stat_path = File.join(path(storage_name), '.') + begin + _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} })) + status == 0 + rescue Errno::ENOENT + File.exist?(stat_path) && File::Stat.new(stat_path).readable? + end + end + + def storage_write_test(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin| + stdin.write(RANDOM_STRING) + end + status == 0 + rescue Errno::ENOENT + written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT + written_bytes == RANDOM_STRING.length + end + + def storage_read_test(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin| + stdin.write(RANDOM_STRING) + end + status == 0 + rescue Errno::ENOENT + file_contents = File.read(tmp_path) rescue Errno::ENOENT + file_contents == RANDOM_STRING + end + + def delete_test_file(tmp_path) + _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} })) + status == 0 + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + end + end + end +end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb new file mode 100644 index 00000000000..1a2eab0b005 --- /dev/null +++ b/lib/gitlab/health_checks/metric.rb @@ -0,0 +1,3 @@ +module Gitlab::HealthChecks + Metric = Struct.new(:name, :value, :labels) +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb new file mode 100644 index 00000000000..57bbe5b3ad0 --- /dev/null +++ b/lib/gitlab/health_checks/redis_check.rb @@ -0,0 +1,25 @@ +module Gitlab + module HealthChecks + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis.with(&:ping) + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb new file mode 100644 index 00000000000..8086760023e --- /dev/null +++ b/lib/gitlab/health_checks/result.rb @@ -0,0 +1,3 @@ +module Gitlab::HealthChecks + Result = Struct.new(:success, :message, :labels) +end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb new file mode 100644 index 00000000000..fbe1645c1b1 --- /dev/null +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -0,0 +1,43 @@ +module Gitlab + module HealthChecks + module SimpleAbstractCheck + include BaseAbstractCheck + + def readiness + check_result = check + if is_successful?(check_result) + HealthChecks::Result.new(true) + elsif check_result.is_a?(Timeout::Error) + HealthChecks::Result.new(false, "#{human_name} check timed out") + else + HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}") + end + end + + def metrics + with_timing method(:check) do |result, elapsed| + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency", elapsed) + ] + end + end + + private + + def metric_prefix + raise NotImplementedError + end + + def is_successful?(result) + raise NotImplementedError + end + + def check + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d787d5db4a0..83bc230df3e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -13,6 +13,8 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end + attr_reader :blob_name + def initialize(blob_name, blob_content, repository: nil) @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @@ -21,16 +23,9 @@ module Gitlab end def highlight(text, continue: true, plain: false) - if plain - hl_lexer = Rouge::Lexers::PlainText - continue = false - else - hl_lexer = self.lexer - end - - @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe - rescue - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + highlighted_text = highlight_text(text, continue: continue, plain: plain) + highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text end def lexer @@ -50,5 +45,27 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end + + def highlight_text(text, continue: true, plain: false) + if plain + highlight_plain(text) + else + highlight_rich(text, continue: continue) + end + end + + def highlight_plain(text) + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + def highlight_rich(text, continue: true) + @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + rescue + highlight_plain(text) + end + + def link_dependencies(text, highlighted_text) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb new file mode 100644 index 00000000000..3411516319f --- /dev/null +++ b/lib/gitlab/i18n.rb @@ -0,0 +1,26 @@ +module Gitlab + module I18n + extend self + + AVAILABLE_LANGUAGES = { + 'en' => 'English', + 'es' => 'Español', + 'de' => 'Deutsch' + }.freeze + + def available_locales + AVAILABLE_LANGUAGES.keys + end + + def set_locale(current_user) + requested_locale = current_user&.preferred_language || ::I18n.default_locale + locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = locale + end + + def reset_locale + FastGettext.set_locale(::I18n.default_locale) + ::I18n.locale = ::I18n.default_locale + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8b327cfc226..27d5a9198b6 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.6'.freeze + VERSION = '0.1.7'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb new file mode 100644 index 00000000000..d4adeeb3797 --- /dev/null +++ b/lib/gitlab/import_export/hash_util.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class HashUtil + def self.deep_symbolize_array!(array) + return if array.blank? + + array.map! do |hash| + hash.deep_symbolize_keys! + + yield(hash) if block_given? + + hash + end + end + + def self.deep_symbolize_array_with_date!(array) + self.deep_symbolize_array!(array) do |hash| + hash.select { |k, _v| k.to_s.end_with?('_date') }.each do |key, value| + hash[key] = Time.zone.parse(value) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ab74c8782f6..d0f3cf2b514 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -40,20 +40,18 @@ project_tree: - :events - :statuses - :triggers - - :deploy_keys + - :pipeline_schedules - :services - :hooks - protected_branches: - :merge_access_levels - :push_access_levels + - protected_tags: + - :create_access_levels - :project_feature # Only include the following attributes for the models specified. included_attributes: - project: - - :description - - :visibility_level - - :archived user: - :id - :email @@ -63,6 +61,32 @@ included_attributes: # Do not include the following attributes for the models specified. excluded_attributes: + project: + - :name + - :path + - :namespace_id + - :creator_id + - :import_url + - :import_status + - :avatar + - :import_type + - :import_source + - :import_error + - :mirror + - :runners_token + - :repository_storage + - :repository_read_only + - :lfs_enabled + - :import_jid + - :created_at + - :updated_at + - :import_jid + - :import_jid + - :id + - :star_count + - :last_activity_at + - :last_repository_updated_at + - :last_repository_check_at snippets: - :expired_at merge_request_diff: @@ -84,8 +108,11 @@ methods: - :type statuses: - :type - - :gl_project_id services: - :type merge_request_diff: - :utf8_st_diffs + merge_requests: + - :diff_head_sha + project: + - :description_html diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 063ce74ecad..fbdd74788bc 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab end def execute - if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb new file mode 100644 index 00000000000..c20adc20bfd --- /dev/null +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -0,0 +1,41 @@ +module Gitlab + module ImportExport + class MergeRequestParser + FORKED_PROJECT_ID = -1 + + def initialize(project, diff_head_sha, merge_request, relation_hash) + @project = project + @diff_head_sha = diff_head_sha + @merge_request = merge_request + @relation_hash = relation_hash + end + + def parse! + if fork_merge_request? && @diff_head_sha + @merge_request.source_project_id = @relation_hash['project_id'] + + fetch_ref unless branch_exists?(@merge_request.source_branch) + create_target_branch unless branch_exists?(@merge_request.target_branch) + end + + @merge_request + end + + def create_target_branch + @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha) + end + + def fetch_ref + @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch) + end + + def branch_exists?(branch_name) + @project.repository.branch_exists?(branch_name) + end + + def fork_merge_request? + @relation_hash['source_project_id'] == FORKED_PROJECT_ID + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index cda6ddf0443..84ab1977dfa 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -52,7 +52,11 @@ module Gitlab create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + relation_hash_list = @tree_hash[relation_key.to_s] + + next unless relation_hash_list + + relation_hash = create_relation(relation_key, relation_hash_list) saved << restored_project.append_or_update_attribute(relation_key, relation_hash) end saved.all? @@ -67,14 +71,14 @@ module Gitlab def restore_project return @project unless @tree_hash - @project.update(project_params) + @project.update_columns(project_params) @project end def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations - value.is_a?(Array) || key == key.singularize + value.respond_to?(:each) && !Project.column_names.include?(key) end end @@ -119,7 +123,7 @@ module Gitlab relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, user: @user, - project_id: restored_project.id) + project: restored_project) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index a1e7159fe42..eb7f5120592 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -15,7 +15,10 @@ module Gitlab # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + attributes = @attributes_finder.find(:project) + project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} + + project_attributes.merge(include: build_hash(@tree)) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d44563333a5..19e23a4715f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -5,15 +5,17 @@ module Gitlab pipelines: 'Ci::Pipeline', statuses: 'commit_status', triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -29,11 +31,12 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id) @members_mapper = members_mapper @user = user + @project = project @imported_object_retries = 0 end @@ -66,7 +69,7 @@ module Gitlab remove_encrypted_attributes! @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] - set_st_diffs if @relation_name == :merge_request_diff + set_st_diff_commits if @relation_name == :merge_request_diff end def update_user_references @@ -105,6 +108,8 @@ module Gitlab imported_object do |object| object.commit_id = nil end + elsif @relation_name == :merge_requests + MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else imported_object end @@ -115,7 +120,7 @@ module Gitlab # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1 + @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID end # project_id may not be part of the export, but we always need to populate it if required. @@ -166,6 +171,7 @@ module Gitlab def imported_object yield(existing_or_new_object) if block_given? existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + existing_or_new_object rescue ActiveRecord::RecordNotUnique # as the operation is not atomic, retry in the unlikely scenario an INSERT is @@ -180,7 +186,7 @@ module Gitlab end def admin_user? - @user.is_admin? + @user.admin? end def parsed_relation_hash @@ -188,8 +194,11 @@ module Gitlab relation_class: relation_class) end - def set_st_diffs + def set_st_diff_commits @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') + + HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) + HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end def existing_or_new_object diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 3a7af363548..4a6091488c8 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -38,7 +38,7 @@ module Gitlab url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], headers: Hash.new { |h, k| h[k] = [] }, - created_at: created_at, + created_at: created_at } end end @@ -64,7 +64,7 @@ module Gitlab tty: true, stdin: true, stdout: true, - stderr: true, + stderr: true }.to_query + '&' + EXEC_COMMAND case url.scheme diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 28129198438..6fdf68641e2 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -39,7 +39,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption, + encryption: encryption ) opts.merge!(auth_options) if has_auth? @@ -124,9 +124,9 @@ module Gitlab def name_proc if allow_username_or_email_login - Proc.new { |name| name.gsub(/@.*\z/, '') } + proc { |name| name.gsub(/@.*\z/, '') } else - Proc.new { |name| name } + proc { |name| name } end end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..49285e35251 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,6 +1,11 @@ module Gitlab module MarkupHelper - module_function + extend self + + MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze + ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze + OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -8,10 +13,7 @@ module Gitlab # # Returns boolean def markup?(filename) - gitlab_markdown?(filename) || - asciidoc?(filename) || - filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki - .mediawiki .rst)) + EXTENSIONS.include?(extension(filename)) end # Public: Determines if a given filename is compatible with @@ -21,7 +23,7 @@ module Gitlab # # Returns boolean def gitlab_markdown?(filename) - filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown)) + MARKDOWN_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename has AsciiDoc extension. @@ -30,7 +32,7 @@ module Gitlab # # Returns boolean def asciidoc?(filename) - filename.downcase.end_with?(*%w(.adoc .ad .asciidoc)) + ASCIIDOC_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename is plain text. @@ -39,12 +41,17 @@ module Gitlab # # Returns boolean def plain?(filename) - filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + extension(filename) == 'txt' || filename.casecmp('readme').zero? end def previewable?(filename) markup?(filename) end + + private + + def extension(filename) + File.extname(filename).downcase.delete('.') + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..cb8db2f1e9f 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -49,6 +49,9 @@ module Gitlab end end end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) end def self.prepare_metrics(metrics) @@ -138,6 +141,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +157,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f98481c6d3a..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::CreateService.new(nil, user_params).build + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index e67acf28c94..31a24460f0f 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -4,21 +4,13 @@ module Gitlab # Public: Converts the provided markup into HTML. # # input - the source text in a markup format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref # def self.render(file_name, input, context) html = GitHub::Markup.render(file_name, input). force_encoding(input.encoding) + context[:pipeline] = :markup - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb new file mode 100644 index 00000000000..f0c50584f07 --- /dev/null +++ b/lib/gitlab/polling_interval.rb @@ -0,0 +1,22 @@ +module Gitlab + class PollingInterval + include Gitlab::CurrentSettings + + HEADER_NAME = 'Poll-Interval'.freeze + + def self.set_header(response, interval:) + if polling_enabled? + multiplier = current_application_settings.polling_interval_multiplier + value = (interval * multiplier).to_i + else + value = -1 + end + + response.headers[HEADER_NAME] = value.to_s + end + + def self.polling_enabled? + !current_application_settings.polling_interval_multiplier.zero? + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index db325c00705..561aa9e162c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -62,7 +62,7 @@ module Gitlab data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') end - OpenStruct.new( + FoundBlob.new( filename: filename, basename: basename, ref: ref, @@ -82,26 +82,14 @@ module Gitlab private def blobs - @blobs ||= begin - blobs = project.repository.search_files_by_content(query, repository_ref).first(100) - found_file_names = Set.new - - results = blobs.map do |blob| - blob = self.class.parse_search_result(blob) - found_file_names << blob.filename - - [blob.filename, blob] - end - - project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| - results << [filename, nil] unless found_file_names.include?(filename) - end + return [] unless Ability.allowed?(@current_user, :download_code, @project) - results.sort_by(&:first) - end + @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) end def wiki_blobs + return [] unless Ability.allowed?(@current_user, :read_wiki, @project) + @wiki_blobs ||= begin if project.wiki_enabled? && query.present? project_wiki = ProjectWiki.new(project) diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb new file mode 100644 index 00000000000..2a2eb4ae57f --- /dev/null +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -0,0 +1,26 @@ +module Gitlab + module Prometheus + module Queries + class BaseQuery + attr_accessor :client + delegate :query_range, :query, to: :client, prefix: true + + def raw_memory_usage_query(environment_slug) + %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} + end + + def raw_cpu_usage_query(environment_slug) + %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} + end + + def initialize(client) + @client = client + end + + def query(*args) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb new file mode 100644 index 00000000000..2cc08731f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -0,0 +1,26 @@ +module Gitlab::Prometheus::Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug + + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), + + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb new file mode 100644 index 00000000000..01d756d7284 --- /dev/null +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -0,0 +1,20 @@ +module Gitlab::Prometheus::Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus_client.rb index 62239779454..5b51a1779dd 100644 --- a/lib/gitlab/prometheus.rb +++ b/lib/gitlab/prometheus_client.rb @@ -2,7 +2,7 @@ module Gitlab PrometheusError = Class.new(StandardError) # Helper methods to interact with Prometheus network services & resources - class Prometheus + class PrometheusClient attr_reader :api_url def initialize(api_url:) @@ -13,18 +13,18 @@ module Gitlab json_api_get('query', query: '1') end - def query(query) + def query(query, time: Time.now) get_result('vector') do - json_api_get('query', query: query) + json_api_get('query', query: query, time: time.to_f) end end - def query_range(query, start: 8.hours.ago) + def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', query: query, start: start.to_f, - end: Time.now.utc.to_f, + end: stop.to_f, step: 1.minute.to_i) end end @@ -50,6 +50,12 @@ module Gitlab def get(url) handle_response(HTTParty.get(url)) + rescue SocketError + raise PrometheusError, "Can't connect to #{url}" + rescue OpenSSL::SSL::SSLError + raise PrometheusError, "#{url} contains invalid SSL data" + rescue HTTParty::Error + raise PrometheusError, "Network connection error" end def handle_response(response) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 5e5f5ff1589..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end @@ -73,22 +77,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def file_path_regex - @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze - end - - def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." - end - - def directory_traversal_regex - @directory_traversal_regex ||= /\.{2}/.freeze - end - - def directory_traversal_regex_message - "cannot include directory traversal." - end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze @@ -121,6 +109,13 @@ module Gitlab git_reference_regex end + ## + # Docker Distribution Registry 2.4.1 repository name rules + # + def container_repository_name_regex + @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + end + def environment_name_regex @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 4b1d828c45c..878e03f61d7 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -2,18 +2,29 @@ module Gitlab module RepoPath NotFoundError = Class.new(StandardError) - def self.strip_storage_path(repo_path) - result = nil + def self.parse(repo_path) + project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) + project = Project.find_by_full_path(project_path) + if project_path.end_with?('.wiki') && !project + project = Project.find_by_full_path(project_path.chomp('.wiki')) + wiki = true + else + wiki = false + end + + [project, wiki] + end + + def self.strip_storage_path(repo_path, fail_on_not_found: true) + result = repo_path - Gitlab.config.repositories.storages.values.each do |params| - storage_path = params['path'] - if repo_path.start_with?(storage_path) - result = repo_path.sub(storage_path, '') - break - end + storage = Gitlab.config.repositories.storages.values.find do |params| + repo_path.start_with?(params['path']) end - if result.nil? + if storage + result = result.sub(storage['path'], '') + elsif fail_on_not_found raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index ccfa517e04b..efe8095beea 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,5 +1,26 @@ module Gitlab class SearchResults + class FoundBlob + attr_reader :id, :filename, :basename, :ref, :startline, :data + + def initialize(opts = {}) + @id = opts.fetch(:id, nil) + @filename = opts.fetch(:filename, nil) + @basename = opts.fetch(:basename, nil) + @ref = opts.fetch(:ref, nil) + @startline = opts.fetch(:startline, nil) + @data = opts.fetch(:data, nil) + end + + def path + filename + end + + def no_highlighting? + false + end + end + attr_reader :current_user, :query # Limit search results by passed projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 117fc508135..2442c2ded3b 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -11,7 +11,7 @@ module Gitlab Raven.user_context( id: current_user.id, email: current_user.email, - username: current_user.username, + username: current_user.username ) end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index da8d8ddb8ed..b1d6ea665b7 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -35,7 +35,7 @@ module Gitlab end def strip_key(key) - key.split(/ /)[0, 2].join(' ') + key.split(/[ ]+/)[0, 2].join(' ') end private @@ -83,7 +83,27 @@ module Gitlab # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, '800']) + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) + raise Error, output unless status.zero? + true + end + + # Fetch remote for repository + # + # name - project path with namespace + # remote - remote name + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # + # Ex. + # fetch_remote("gitlab/gitlab-ci", "upstream") + # + def fetch_remote(storage, name, remote, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] + args << '--force' if forced + args << '--no-tags' if no_tags + + output, status = Popen.popen(args) raise Error, output unless status.zero? true end @@ -174,7 +194,10 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - FileUtils.mkdir_p(full_path(storage, name), mode: 0770) unless exists?(storage, name) + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + rescue Errno::EEXIST => e + Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") end # Remove directory from repositories storage diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 11e5f1b645c..ca8d3271541 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -72,6 +72,8 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. + # true = job is still running + # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } @@ -82,6 +84,17 @@ module Gitlab end end + # Returns the JIDs that are completed + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of completed JIDs + def self.completed_jids(job_ids) + Sidekiq.redis do |redis| + job_ids.reject { |jid| redis.exists(key_for(jid)) } + end + end + def self.key_for(jid) STATUS_KEY % jid end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index d47609f490d..00983b3284a 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -2,7 +2,9 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - Gitlab::SidekiqStatus.set(job['jid']) + status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + + Gitlab::SidekiqStatus.set(job['jid'], status_expiration) yield end end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb index 60d35be2599..12a385f90fd 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -1,16 +1,19 @@ module Gitlab module SlashCommands class CommandDefinition - attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + attr_accessor :name, :aliases, :description, :explanation, :params, + :condition_block, :parse_params_block, :action_block def initialize(name, attributes = {}) @name = name - @aliases = attributes[:aliases] || [] - @description = attributes[:description] || '' - @params = attributes[:params] || [] + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @explanation = attributes[:explanation] || '' + @params = attributes[:params] || [] @condition_block = attributes[:condition_block] - @action_block = attributes[:action_block] + @parse_params_block = attributes[:parse_params_block] + @action_block = attributes[:action_block] end def all_names @@ -28,14 +31,20 @@ module Gitlab context.instance_exec(&condition_block) end + def explain(context, opts, arg) + return unless available?(opts) + + if explanation.respond_to?(:call) + execute_block(explanation, context, arg) + else + explanation + end + end + def execute(context, opts, arg) return if noop? || !available?(opts) - if arg.present? - context.instance_exec(arg, &action_block) - elsif action_block.arity == 0 - context.instance_exec(&action_block) - end + execute_block(action_block, context, arg) end def to_h(opts) @@ -52,6 +61,23 @@ module Gitlab params: params } end + + private + + def execute_block(block, context, arg) + if arg.present? + parsed = parse_params(arg, context) + context.instance_exec(parsed, &block) + elsif block.arity == 0 + context.instance_exec(&block) + end + end + + def parse_params(arg, context) + return arg unless parse_params_block + + context.instance_exec(arg, &parse_params_block) + end end end end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb index 50b0937d267..614bafbe1b2 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/slash_commands/dsl.rb @@ -44,6 +44,22 @@ module Gitlab @params = params end + # Allows to give an explanation of what the command will do when + # executed. This explanation is shown when rendering the Markdown + # preview. + # + # Example: + # + # explanation do |arguments| + # "Adds label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def explanation(text = '', &block) + @explanation = block_given? ? block : text + end + # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # It accepts a block that will be evaluated with the context given to @@ -61,6 +77,24 @@ module Gitlab @condition_block = block end + # Allows to perform initial parsing of parameters. The result is passed + # both to `command` and `explanation` blocks, instead of the raw + # parameters. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # parse_params do |raw| + # raw.strip + # end + # command :command_key do |parsed| + # # Awesome code block + # end + def parse_params(&block) + @parse_params_block = block + end + # Registers a new command which is recognizeable from body of email or # comment. # It accepts aliases and takes a block. @@ -75,11 +109,13 @@ module Gitlab definition = CommandDefinition.new( name, - aliases: aliases, - description: @description, - params: @params, - condition_block: @condition_block, - action_block: block + aliases: aliases, + description: @description, + explanation: @explanation, + params: @params, + condition_block: @condition_block, + parse_params_block: @parse_params_block, + action_block: block ) self.command_definitions << definition @@ -89,8 +125,14 @@ module Gitlab end @description = nil + @explanation = nil @params = nil @condition_block = nil + @parse_params_block = nil + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] end end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb new file mode 100644 index 00000000000..94fba0a221a --- /dev/null +++ b/lib/gitlab/string_range_marker.rb @@ -0,0 +1,102 @@ +module Gitlab + class StringRangeMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(marker_ranges) + return rich_line unless marker_ranges + + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + + offset = 0 + # Mark each range + rich_marker_ranges.each_with_index do |range, i| + offset_range = (range.begin + offset)..(range.end + offset) + original_text = rich_line[offset_range] + + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + + rich_line[offset_range] = text + + offset += text.length - original_text.length + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + end +end diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb new file mode 100644 index 00000000000..7ebf1c0428c --- /dev/null +++ b/lib/gitlab/string_regex_marker.rb @@ -0,0 +1,13 @@ +module Gitlab + class StringRegexMarker < StringRangeMarker + def mark(regex, group: 0, &block) + regex_match = raw_line.match(regex) + return rich_line unless regex_match + + begin_index, end_index = regex_match.offset(group) + name_range = begin_index..(end_index - 1) + + super([name_range], &block) + end + end +end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..bcba2e3e1b6 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,66 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + ci_pipeline_schedules: ::Ci::PipelineSchedule.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index f260c0c535f..3b922da7ced 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -28,16 +28,33 @@ module Gitlab true end + def can_create_tag?(ref) + return false unless can_access_git? + + if ProtectedTag.protected?(project, ref) + project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + else + user.can?(:push_code, project) + end + end + + def can_delete_branch?(ref) + return false unless can_access_git? + + if ProtectedBranch.protected?(project, ref) + user.can?(:delete_protected_branch, project) + else + user.can?(:push_code, project) + end + end + def can_push_to_branch?(ref) return false unless can_access_git? - if project.protected_branch?(ref) + if ProtectedBranch.protected?(project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - has_access = access_levels.any? { |access_level| access_level.check_access(user) } - - has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) else user.can?(:push_code, project) end @@ -46,9 +63,8 @@ module Gitlab def can_merge_to_branch?(ref) return false unless can_access_git? - if project.protected_branch?(ref) - access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + if ProtectedBranch.protected?(project, ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) else user.can?(:push_code, project) end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 8f1d1fdc02e..2e31f4462f9 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -63,7 +63,7 @@ module Gitlab end def allowed_for?(user, level) - user.is_admin? || allowed_level?(level.to_i) + user.admin? || allowed_level?(level.to_i) end # Return true if the specified level is allowed for the current user. diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 6fe85af3c30..fe37e4da94f 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -16,15 +16,32 @@ module Gitlab SECRET_LENGTH = 32 class << self - def git_http_ok(repository, user) + def git_http_ok(repository, is_wiki, user, action) + project = repository.project + repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), - RepoPath: repository.path_to_repo, + GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + RepoPath: repo_path } if Gitlab.config.gitaly.enabled - address = Gitlab::GitalyClient.get_address(repository.project.repository_storage) - params[:GitalySocketPath] = URI(address).path + address = Gitlab::GitalyClient.address(project.repository_storage) + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 + false + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + + params[:GitalyAddress] = address if feature_enabled end params @@ -34,7 +51,7 @@ module Gitlab { StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", LfsOid: oid, - LfsSize: size, + LfsSize: size } end @@ -45,7 +62,7 @@ module Gitlab def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id, + 'BlobId' => blob.id } [ @@ -110,7 +127,7 @@ module Gitlab 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], 'Header' => terminal[:headers], - 'MaxSessionTime' => terminal[:max_session_time], + 'MaxSessionTime' => terminal[:max_session_time] } } details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) @@ -148,12 +165,12 @@ module Gitlab encoded_message, secret, true, - { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' } ) end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb new file mode 100644 index 00000000000..d2c420efdaf --- /dev/null +++ b/lib/microsoft_teams/activity.rb @@ -0,0 +1,19 @@ +module MicrosoftTeams + class Activity + def initialize(title:, subtitle:, text:, image:) + @title = title + @subtitle = subtitle + @text = text + @image = image + end + + def prepare + { + 'activityTitle' => @title, + 'activitySubtitle' => @subtitle, + 'activityText' => @text, + 'activityImage' => @image + } + end + end +end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb new file mode 100644 index 00000000000..3bef68a1bcb --- /dev/null +++ b/lib/microsoft_teams/notifier.rb @@ -0,0 +1,46 @@ +module MicrosoftTeams + class Notifier + def initialize(webhook) + @webhook = webhook + @header = { 'Content-type' => 'application/json' } + end + + def ping(options = {}) + result = false + + begin + response = HTTParty.post( + @webhook.to_str, + headers: @header, + body: body(options) + ) + + result = true if response + rescue HTTParty::Error, StandardError => error + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + end + + result + end + + private + + def body(options = {}) + result = { 'sections' => [] } + + result['title'] = options[:title] + result['summary'] = options[:pretext] + result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare + + attachments = options[:attachments] + unless attachments.blank? + result['sections'] << { + 'title' => 'Details', + 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] + } + end + + result.to_json + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 09e121e5120..6e351365de0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -326,8 +326,7 @@ start_gitlab() { echo "Gitaly is already running with pid $gapid, not restarting" else $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ - $app_root/bin/with_env $gitaly_dir/env \ - $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 & fi fi diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 2301ec9b228..99b3168d9eb 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 5293f5af12d..87ca39b079b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -19,8 +19,9 @@ namespace :gemojione do entry = { category: emoji_hash['category'], moji: emoji_hash['moji'], + description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), - digest: hash_digest, + digest: hash_digest } resultant_emoji_map[name] = entry diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake new file mode 100644 index 00000000000..0aa21a4bd13 --- /dev/null +++ b/lib/tasks/gettext.rake @@ -0,0 +1,14 @@ +require "gettext_i18n_rails/tasks" + +namespace :gettext do + # Customize list of translatable files + # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files + def files_to_translate + folders = %W(app lib config #{locale_path}).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + + Dir.glob( + "{#{folders}}/**/*.{#{exts}}" + ) + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a9a48f7188f..f41c73154f5 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,8 +431,7 @@ namespace :gitlab do def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" + puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| repo_base_path = repository_storage['path'] @@ -443,15 +442,16 @@ namespace :gitlab do break end - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + user_id = uid_for(gitlab_shell_ssh_user) + root_group_id = gid_for('root') + group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] + if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) puts "yes".color(:green) else puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" ) for_more_information( see_installation_guide_section "GitLab Shell" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 5476438b8fa..139ab70e125 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -65,6 +65,7 @@ namespace :gitlab do migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines .map { |file| Rails.root.join(file.strip).to_s } .select { |file| File.file?(file) } + .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index c288e17ac8d..3c5bc0146a1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,23 +1,74 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| + require 'toml' + warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') - tag = "v#{Gitlab::GitalyClient.expected_server_version}" - repo = 'https://gitlab.com/gitlab-org/gitaly.git' + version = Gitlab::GitalyClient.expected_server_version - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do + create_gitaly_configuration run_command!([command]) end end + + desc "GitLab | Print storage configuration in TOML format" + task storage_config: :environment do + require 'toml' + + puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" + puts "# This is in TOML format suitable for use in Gitaly's config.toml file." + + config = Gitlab.config.repositories.storages.map do |key, val| + { name: key, path: val['path'] } + end + + puts TOML.dump(storage: config) + end + + private + + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only write a config.toml file in the most common and simplest case: the + # case where we have exactly one Gitaly process and we are sure it is + # running locally because it uses a Unix socket. + def create_gitaly_configuration + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + File.open("config.toml", "w") do |f| + f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + end + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index a2a2db487b7..e3883278886 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -16,6 +16,8 @@ namespace :gitlab do redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a # check Git version git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a + # check Go version + go_version = run_and_match(%w(go version), /go version (.+)/).to_a puts "" puts "System information".color(:yellow) @@ -30,6 +32,7 @@ namespace :gitlab do puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" + puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd2fda54e62..ee2cdcdea1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,19 +1,18 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:tag, :repo] => :environment do |t, args| + task :install, [:repo] => :environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = "v#{default_version}" - args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do @@ -42,8 +41,14 @@ namespace :gitlab do # Generate config.yml based on existing gitlab settings File.open("config.yml", "w+") {|f| f.puts config.to_yaml} - # Launch installation process - system(*%w(bin/install) + repository_storage_paths_args) + [ + %w(bin/install) + repository_storage_paths_args, + %w(bin/compile) + ].each do |cmd| + unless Kernel.system(*cmd) + raise "command failed: #{cmd.join(' ')}" + end + end end # (Re)create hooks diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index cdba2262bc2..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -147,41 +147,30 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - checkout_tag(tag, target_dir) - else - clone_repo(repo, target_dir) - end + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end - reset_to_tag(tag, target_dir) + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end - def checkout_tag(tag, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) end - def reset_to_tag(tag_wanted, target_dir) - tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end + def reset_to_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index dbdfb335a5c..59c32bbe7a4 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,7 +5,7 @@ namespace :gitlab do end def update(template) - sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) unless clone_repository(template.repo_url, dir) @@ -44,7 +44,11 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ + ), + Template.new( + "https://gitlab.com/gitlab-org/Dockerfile.git", + /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) ].freeze diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index baea94bf8ca..e7ac0b5859f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,16 +1,16 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') - tag = "v#{Gitlab::Workhorse.version}" - repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + version = Gitlab::Workhorse.version - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 00000000000..bc76d7edc55 --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,142 @@ +require 'benchmark' +require 'rainbow/ext/string' + +class GithubImport + def self.run!(*args) + new(*args).run! + end + + def initialize(token, gitlab_username, project_path, extras) + @options = { url: 'https://api.github.com', token: token, verbose: true } + @project_path = project_path + @current_user = User.find_by_username(gitlab_username) + @github_repo = extras.empty? ? nil : extras.first + end + + def run! + @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! + + raise 'No repo found!' unless @repo + + show_warning! + + @project = Project.find_by_full_path(@project_path) || new_project + + import! + end + + private + + def show_warning! + puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts "Permission checks are ignored. Press any key to continue.".color(:red) + + STDIN.getch + + puts 'Starting the import (this could take a while)'.color(:green) + end + + def import! + @project.import_start + + timings = Benchmark.measure do + Github::Import.new(@project, @options).execute + end + + puts "Import finished. Timings: #{timings}".color(:green) + + @project.import_finish + end + + def new_project + Project.transaction do + namespace_path, _sep, name = @project_path.rpartition('/') + namespace = find_or_create_namespace(namespace_path) + + Projects::CreateService.new( + @current_user, + name: name, + path: name, + description: @repo['description'], + namespace_id: namespace.id, + visibility_level: visibility_level, + import_type: 'github', + import_source: @repo['full_name'], + skip_wiki: @repo['has_wiki'] + ).execute + end + end + + def find_or_create_namespace(names) + return @current_user.namespace if names == @current_user.namespace_path + return @current_user.namespace unless @current_user.can_create_group? + + full_path_namespace = Namespace.find_by_full_path(names) + + return full_path_namespace if full_path_namespace + + names.split('/').inject(nil) do |parent, name| + begin + namespace = Group.create!(name: name, + path: name, + owner: @current_user, + parent: parent) + namespace.add_owner(@current_user) + + namespace + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + Namespace.where(parent: parent).find_by_path_or_name(name) + end + end + end + + def full_path_namespace(names) + @full_path_namespace ||= Namespace.find_by_full_path(names) + end + + def visibility_level + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end +end + +class GithubRepos + def initialize(options, current_user, github_repo) + @options = options + @current_user = current_user + @github_repo = github_repo + end + + def choose_one! + return found_github_repo if @github_repo + + repos.each do |repo| + print "ID: #{repo['id'].to_s.bright}".color(:green) + print "\tName: #{repo['full_name']}\n".color(:green) + end + + print 'ID? '.bright + + repos.find { |repo| repo['id'] == repo_id } + end + + def found_github_repo + repos.find { |repo| repo['full_name'] == @github_repo } + end + + def repo_id + @repo_id ||= STDIN.gets.chomp.to_i + end + + def repos + Github::Repositories.new(@options).fetch + end +end + +namespace :import do + desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)' + task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args| + abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/') + + GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras) + end +end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 6ded519aff2..761f275d42a 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,7 +1,9 @@ require Rails.root.join('db/migrate/limits_to_mysql') +require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do puts "Adding limits to schema.rb for mysql" LimitsToMysql.new.up + MarkdownCacheLimitsToMysql.new.up end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 8938bc515f5..4108cee08b4 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') +require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do @@ -11,4 +12,5 @@ task setup_postgresql: :environment do AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up IndexRoutesPathForLike.new.up + IndexRedirectRoutesPathForLike.new.up end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 602c60be828..2eddcb3c777 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -60,7 +60,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %w(rake gitlab:setup), - %w(rspec spec), + %w(rspec spec) ] run_commands(cmds) end |